Observability

{
  "channels": {
    "ldap": {
      "enabled": true,
      "running": true,
      "bindings": [
        {
          "host": "0.0.0.0",
          "configuredPort": 0,
          "actualPort": 10389,
          "ssl": false,
          "url": "ldap://0.0.0.0:10389"
        },
        {
          "host": "0.0.0.0",
          "configuredPort": 10636,
          "actualPort": 10636,
          "ssl": true,
          "url": "ldaps://0.0.0.0:10636"
        }
      ]
    },
    "identityHub": {
      "enabled": false,
      "running": false,
      "bindings": []
    },
    "rest": {
      "enabled": true,
      "sockets": ["@default"],
      "basePath": "/api"
    }
  },
  "monitoring": {
    "prometheus": { "enabled": true, "sockets": ["@default"], "path": "/metrics" },
    "observe": { "enabled": true, "sockets": ["@default"], "path": "/observe" },
    "health": { "enabled": true, "sockets": ["@default"], "paths": ["/livez", "/readyz", "/startedz", "/healthz"] }
  },
  "telemetry": {
    "traces": { "enabled": true },
    "metrics": { "prometheus": true, "otlp": false },
    "hints": { "enabled": true, "explain": false, "persistence": true }
  },
  "sockets": {
    "@default": {
      "host": "0.0.0.0",
      "configuredPort": 0,
      "actualPort": 8080,
      "ssl": false,
      "url": "http://api.example.com:8080"
    },
    "internal": {
      "host": "127.0.0.1",
      "configuredPort": 9090,
      "actualPort": 9090,
      "ssl": false,
      "url": "http://127.0.0.1:9090"
    }
  },
  "request": {
    "socket": "@default",
    "host": "api.example.com",
    "port": 8080,
    "scheme": "http"
  },
  "timestamp": "2026-01-13T12:00:00Z"
}

{
  "config": "Configuration Sources:\n\n- system properties\n- reference.conf\n\napp {\n  mode = production\n}\n\ndatabase {\n  \"password\" : \"<REDACTED>\"\n  host = \"localhost\"\n  port = 5432\n}\n...",
  "timestamp": "2026-01-13T12:00:00Z"
}

{
  "services": [
    {
      "id": "Scribe.user",
      "name": "Scribe",
      "state": "running",
      "healthy": true,
      "tags": {"entryType": "user"},
      "uptime_seconds": 3600,
      "startup_seconds": 2.5
    },
    {
      "id": "Database.System",
      "name": "Database.System",
      "state": "failed",
      "healthy": false,
      "failure": "Connection refused"
    }
  ],
  "summary": {
    "total": 8,
    "healthy": 7,
    "unhealthy": 1,
    "restarts_5m": 3
  },
  "timestamp": "2026-01-08T12:00:00Z"
}

# Query signals alerting
scribe_signals_latency_p95 > 2.0
scribe_signals_error_rate_percent > 5.0
scribe_signals_traffic_ratio > 5.0

# Ingest signals alerting
scribe_signals_ingest_lag_max_seconds > 300
scribe_signals_ingest_failed_rate_percent > 1.0
scribe_signals_ingest_task_duration_p95 > 5.0

# Per-entry-type lag comparison
scribe_signals_ingest_entry_lag_seconds

# Request rate by channel
rate(scribe_channel_requests_total[5m])

# p99 latency by channel
histogram_quantile(0.99, rate(scribe_channel_request_duration_seconds_bucket[5m]))

# Error rate
sum(rate(scribe_channel_requests_total{result!="ok"}[5m]))
  / sum(rate(scribe_channel_requests_total[5m]))

# Query permit pressure sustained high
avg_over_time(scribe_query_permit_pressure[5m]) > 0.9

# Ingest queue pressure by entry_type
scribe_ingest_queue_pressure > 0.8

# Ingest falling behind
scribe_ingest_task_pressure > 1.2

# p95 by stage
histogram_quantile(0.95,
  sum by (stage, le) (rate(scribe_query_stage_duration_seconds_bucket[5m]))
)

# Which stage dominates?
sum by (stage) (rate(scribe_query_stage_duration_seconds_sum[5m]))
  / sum(rate(scribe_query_stage_duration_seconds_sum[5m]))

# Services that restarted recently
increase(scribe_service_restarts_total[1h]) > 0

# Services currently down
scribe_service_up == 0

# Service state transitions (e.g., identify flapping services)
rate(scribe_service_transitions_total[5m])

# How long services spent starting (to detect slow startups)
histogram_quantile(0.95, rate(scribe_service_transition_duration_seconds_bucket{to="running"}[1h]))

# Services that failed recently
increase(scribe_service_transitions_total{to="failed"}[1h]) > 0

LDAP.Search (or REST.Search, etc.)         ← Channel entry span
└── Query.Normalize                         ← Normalization stage
└── Query.Plan                              ← Planning stage
└── Query.Compile                           ← SQL emission stage (Prepare)
└── Query.Execute                           ← DB execution + result streaming

monitoring.telemetry.traces {
  enabled = true
  endpoint = "http://otel-collector:4317"
  protocol = "grpc"  # or "http/protobuf" for port 4318
}

{"trace_id":"...","span_id":"...","duration_seconds":1.5,"result":"ok","scribe.operation":"LDAP.Search",...}

{
  "trace_id": "abc123",
  "span_id": "def456",
  "duration_seconds": 1.5,
  "result": "ok",
  "scribe.operation": "LDAP.Search",
  "scribe.entry_type": "user",
  "scribe.search.kind": "paged",
  "events": [
    {"name": "warning", "timestamp": "...", "attributes": {"code": "unsupported_control", "message": "VLV not available"}}
  ]
}

LDAP.Search ok dur=1.5s warnings=1 events=5
  trace=abc123 span=def456
  db: system=postgresql duration.seconds=0.42 row_count=42
  http: route="/ldap/search"
  scribe: entry_type=user search.kind=paged
  scribe.query: scope="ou=users,o=org" where="(&(objectClass=*))" sort="cn"
  events:
    +  5.1ms  12.3ms  Query.Plan
    warning code=unsupported_control message="VLV not available"
    + 18.0ms 401.9ms  DB.Fetch row_count=42
    cache.hit
    +520.2ms   9.8ms  Limiter.Acquire permits=1

scribe REST.Modify internal dur=311ms trace=trace123 span=span456
  failure:
    kind=INTERNAL code=SERVER_ERROR
    message="Database connection failed"
    details={"context": "user.modify", "attempt": 3}
    trace_id=trace123 span_id=span456
    cause_type=java.sql.SQLException
    cause_message="Connection refused"
    stacktrace:
      java.sql.SQLException: Connection refused
        at org.postgresql.Driver.connect(Driver.java:285)
        at ...
  scribe: entry_type=user

monitoring.log {
  # Enable/disable wide event logging (default: true)
  enabled = true
  enabled = ${?SCRIBE_LOG_ENABLED}

  # Log format: pretty (default) | json | auto
  # auto = pretty in dev mode + TTY, json otherwise
  # Uses app.mode (dev/development/local/test → dev mode)
  format = "pretty"
  format = ${?SCRIBE_LOG_FORMAT}

  # Random sampling for ops that don't match any rule
  # 0 = never sample, 100 = always log (default)
  sample-rate = 100
  sample-rate = ${?SCRIBE_LOG_SAMPLE_RATE}

  # Per-key redaction strategies
  # Keys support glob patterns: * = any chars, ? = single char
  #
  # Available strategies:
  #   replace  - Replace with "[REDACTED]"
  #   hash     - SHA-256, base64url, 16 chars
  #   truncate - Show ≤33% of chars (max 8 visible); ≤8 chars fall back to [REDACTED]
  #   omit     - Remove attribute entirely
  #
  # Hard-coded security patterns (always OMIT, cannot be disabled):
  #   *password*, *credential*, *token*, *secret*, *apikey*, *api_key*
  redaction {
    "*dn*" = hash           # Matches scribe.entry_dn, ldap.base_dn, etc.
    "*email*" = truncate    # Matches user.email, notification.email, etc.
    "*.raw" = replace
    "*.pii.*" = hash
  }
}

monitoring.log.rules = [
  { action = include|exclude, name = "glob", where = "(filter)" }
]

monitoring.log.rules = [
  # Suppress fast successful operations
  { action = exclude, where = "(&(scribe.result=ok)(duration.seconds<=50ms))" }

  # Always log slow operations
  { action = include, where = "(duration.seconds>=5s)" }

  # Suppress internal maintenance under threshold
  { action = exclude, name = "Hints.*", where = "(duration.seconds<=100ms)" }
  { action = exclude, name = "Metrics.*", where = "(duration.seconds<=500ms)" }
]

# Log all LDAP operations
{ action = include, name = "LDAP.*" }

# Suppress fast successful ops
{ action = exclude, where = "(&(scribe.result=ok)(duration.seconds<=50ms))" }

# Log slow DB queries
{ action = include, where = "(db.duration.seconds>=1s)" }

# Exclude everything else (catch-all)
{ action = exclude }

monitoring.log.childs {
  # Mode: auto (default) | off | minimal | full
  # auto = full in dev mode, minimal in prod
  mode = "auto"
  mode = ${?SCRIBE_LOG_CHILDS_MODE}

  # Display: auto (default) | off | summary | details
  # auto = summary in dev, off in prod
  display = "auto"

  # Rules: which segments to track (first match wins, empty = include all)
  rules = [
    { action = include, where = "(duration.seconds>=1ms)" }
    { action = exclude }  # Catch-all: filter sub-millisecond noise
  ]
}

# Track all segments (no filtering)
childs {
  mode = "full"
  rules = []  # Empty = include all
}

# Track only slow segments
childs {
  mode = "minimal"
  rules = [
    { action = include, name = "Query.*", where = "(duration.seconds>=10ms)" }
    { action = include, name = "DB.*", where = "(duration.seconds>=5ms)" }
    { action = exclude }
  ]
}

{
  "duration_seconds": 0.125,
  "result": "ok",
  "scribe.operation": "LDAP.Search",
  "events": [
    {
      "name": "Query.Plan",
      "timestamp": "2026-01-07T10:30:00.001Z",
      "attributes": { "offset.seconds": 0.001, "duration.seconds": 0.002 }
    },
    {
      "name": "Query.Execute",
      "timestamp": "2026-01-07T10:30:00.003Z",
      "attributes": { "offset.seconds": 0.003, "duration.seconds": 0.095 }
    },
    {
      "name": "Query.Map",
      "timestamp": "2026-01-07T10:30:00.098Z",
      "attributes": { "offset.seconds": 0.098, "duration.seconds": 0.025 }
    }
  ]
}

startupProbe:
  httpGet:
    path: /startedz
    port: 8081
  initialDelaySeconds: 5
  periodSeconds: 10
  failureThreshold: 90   # 15min max for services + DB init

livenessProbe:
  httpGet:
    path: /livez
    port: 8081
  initialDelaySeconds: 0
  periodSeconds: 10
  failureThreshold: 3

readinessProbe:
  httpGet:
    path: /readyz
    port: 8081
  initialDelaySeconds: 0
  periodSeconds: 5
  failureThreshold: 1
  successThreshold: 1