diff --git a/Dockerfile b/Dockerfile index a0e3299..f37da8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,21 @@ # syntax=docker/dockerfile:1.4 # ============================================================================= -# Dockhand Docker Image - Security-Hardened Build +# Dockhand Docker Image - Node.js Runtime (Security-Hardened Build) # ============================================================================= -# This Dockerfile builds a custom Wolfi OS from scratch using apko, ensuring: -# - Full transparency (no dependency on pre-built Chainguard images) -# - Reproducible builds from open-source Wolfi packages -# - Minimal attack surface with only required packages -# -# Bun is copied from the official oven/bun image (app-builder stage). -# For CPUs without AVX support (Celeron, Atom, pre-Haswell), build with: -# docker build --build-arg BUN_VARIANT=baseline -t dockhand:baseline . +# Uses Node.js instead of Bun to eliminate BoringSSL native memory leaks +# on mTLS connections. Same Wolfi-based security-hardened OS. # ============================================================================= # ----------------------------------------------------------------------------- # Stage 1: OS Generator (Alpine + apko tool) # ----------------------------------------------------------------------------- -# We use Alpine because it has a shell. This lets us download and run apko -# to build our custom Wolfi OS from scratch using open-source packages. FROM alpine:3.21 AS os-builder ARG TARGETARCH WORKDIR /work -# Install apko tool (latest stable release) -# apko is the tool Chainguard uses to build their images - we use it directly +# Install apko tool ARG APKO_VERSION=0.30.34 RUN apk add --no-cache curl unzip \ && ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "arm64" || echo "amd64") \ @@ -32,9 +23,7 @@ RUN apk add --no-cache curl unzip \ | tar -xz --strip-components=1 -C /usr/local/bin \ && chmod +x /usr/local/bin/apko -# Generate apko.yaml for current target architecture only -# We build single-arch to avoid multi-arch layer confusion in extraction -# Note: Bun is NOT included here - it's copied from app-builder stage for CPU compatibility +# Generate apko.yaml — Node.js instead of Bun RUN APKO_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "x86_64") \ && printf '%s\n' \ "contents:" \ @@ -47,6 +36,7 @@ RUN APKO_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "x86_64") " - ca-certificates" \ " - busybox" \ " - tzdata" \ + " - nodejs-24" \ " - docker-cli" \ " - docker-compose" \ " - docker-cli-buildx" \ @@ -58,6 +48,8 @@ RUN APKO_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "x86_64") " - curl" \ " - tini" \ " - su-exec" \ + " - glibc" \ + " - libstdc++" \ "entrypoint:" \ " command: /bin/sh -l" \ "archs:" \ @@ -65,7 +57,6 @@ RUN APKO_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "x86_64") > apko.yaml # Build the OS tarball and extract rootfs -# apko creates an OCI tarball - we need to extract the actual filesystem layer RUN apko build apko.yaml dockhand-base:latest output.tar \ && mkdir -p rootfs \ && tar -xf output.tar \ @@ -73,67 +64,46 @@ RUN apko build apko.yaml dockhand-base:latest output.tar \ && tar -xzf "$LAYER" -C rootfs # ----------------------------------------------------------------------------- -# Stage 2: Application Builder +# Stage 2: Application Builder (pure Node.js) # ----------------------------------------------------------------------------- -# Using Debian to avoid Alpine musl thread creation issues -# Alpine's musl libc causes rayon/tokio thread pool panics during svelte-adapter-bun build -FROM oven/bun:1.3.5-debian AS app-builder - -# Build argument for Bun variant (regular or baseline) -# baseline is for CPUs without AVX support (Celeron, Atom, pre-Haswell) -ARG BUN_VARIANT=regular -ARG TARGETARCH +FROM node:24-slim AS app-builder WORKDIR /app # Install build dependencies -# libnss-wrapper: needed for git SSH with arbitrary UIDs on read-only containers (getpwuid workaround) -RUN apt-get update && apt-get install -y --no-install-recommends jq git curl unzip ca-certificates libnss-wrapper && rm -rf /var/lib/apt/lists/* \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + jq git curl python3 make g++ libnss-wrapper \ + && rm -rf /var/lib/apt/lists/* \ && cp "$(dpkg -L libnss-wrapper | grep 'libnss_wrapper\.so$')" /usr/local/lib/libnss_wrapper.so -# Copy package files and install ALL dependencies (needed for build) -COPY package.json bun.lock* bunfig.toml ./ -RUN bun install --frozen-lockfile +# Copy package files and install dependencies +COPY package.json package-lock.json ./ +RUN npm ci # Copy source code and build COPY . . +RUN npm run build -# Build the application -RUN NODE_OPTIONS="--max-old-space-size=8192 --max-semi-space-size=128" bun run build +# Production dependencies only (rebuilds native addons like better-sqlite3) +RUN rm -rf node_modules \ + && npm ci --omit=dev \ + && rm -rf node_modules/@types -# Prepare production node_modules (do this in builder where we have compilers) -# This ensures native addons compile correctly before copying to hardened runtime -RUN rm -rf node_modules && bun install --production --frozen-lockfile \ - && rm -rf node_modules/@types node_modules/bun-types - -# Download baseline Bun binary if BUN_VARIANT=baseline (for CPUs without AVX) -# Only applies to amd64 - ARM64 doesn't have AVX concept -ARG BUN_VERSION=1.3.5 -RUN if [ "$BUN_VARIANT" = "baseline" ] && [ "$TARGETARCH" = "amd64" ]; then \ - echo "Downloading Bun baseline binary for CPUs without AVX support..." && \ - curl -fsSL "https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/bun-linux-x64-baseline.zip" -o /tmp/bun.zip && \ - unzip -o /tmp/bun.zip -d /tmp && \ - cp /tmp/bun-linux-x64-baseline/bun /usr/local/bin/bun && \ - chmod +x /usr/local/bin/bun && \ - rm -rf /tmp/bun.zip /tmp/bun-linux-x64-baseline && \ - echo "Bun baseline binary installed successfully"; \ - fi +# Build Go collector +FROM golang:1.24 AS go-builder +WORKDIR /app +COPY collector/ ./collector/ +RUN cd collector && CGO_ENABLED=0 go build -o /app/bin/collection-worker . # ----------------------------------------------------------------------------- # Stage 3: Final Image (Scratch + Custom Wolfi OS) # ----------------------------------------------------------------------------- FROM scratch -# Install our custom-built Wolfi OS (now we have /bin/sh!) +# Install custom Wolfi OS with Node.js COPY --from=os-builder /work/rootfs/ / -# Copy Bun from official image - ensures compatibility with all x86_64 CPUs (no AVX2 requirement) -# Wolfi's bun package requires AVX2 which breaks on Celeron/Atom CPUs -# For baseline builds (BUN_VARIANT=baseline), this contains the baseline binary (no AVX requirement) -# For regular builds, this contains the standard oven/bun binary -COPY --from=app-builder /usr/local/bin/bun /usr/bin/bun - -# Copy libnss_wrapper for git SSH with arbitrary UIDs (same cross-copy pattern as Bun above) +# Copy libnss_wrapper for git SSH with arbitrary UIDs COPY --from=app-builder /usr/local/lib/libnss_wrapper.so /usr/lib/libnss_wrapper.so WORKDIR /app @@ -149,20 +119,22 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ PUID=1001 \ PGID=1001 -# Create docker compose plugin symlink (we use `docker compose` syntax, Wolfi has standalone binary) -# Note: docker-cli-buildx package already creates the buildx symlink +# Create docker compose plugin symlink RUN mkdir -p /usr/libexec/docker/cli-plugins \ - && ln -s /usr/bin/docker-compose /usr/libexec/docker/cli-plugins/docker-compose + && ln -sf /usr/bin/docker-compose /usr/libexec/docker/cli-plugins/docker-compose -# Create dockhand user and group (using busybox commands) +# Create dockhand user and group RUN addgroup -g 1001 dockhand \ && adduser -u 1001 -G dockhand -h /home/dockhand -D dockhand -# Copy application files with correct ownership (avoids layer duplication from chown -R) +# Copy application files with correct ownership COPY --from=app-builder --chown=dockhand:dockhand /app/node_modules ./node_modules COPY --from=app-builder --chown=dockhand:dockhand /app/package.json ./ COPY --from=app-builder --chown=dockhand:dockhand /app/build ./build -COPY --from=app-builder --chown=dockhand:dockhand /app/build/subprocesses/ ./subprocesses/ +COPY --from=app-builder --chown=dockhand:dockhand /app/server.js ./ + +# Copy Go collector binary +COPY --from=go-builder --chown=dockhand:dockhand /app/bin/collection-worker ./bin/collection-worker # Copy database migrations COPY --chown=dockhand:dockhand drizzle/ ./drizzle/ @@ -171,15 +143,15 @@ COPY --chown=dockhand:dockhand drizzle-pg/ ./drizzle-pg/ # Copy legal documents COPY --chown=dockhand:dockhand LICENSE.txt PRIVACY.txt ./ -# Copy entrypoint script (root-owned, executable) -COPY docker-entrypoint.sh /usr/local/bin/ +# Copy entrypoint script +COPY docker-entrypoint-node.sh /usr/local/bin/docker-entrypoint.sh RUN chmod +x /usr/local/bin/docker-entrypoint.sh # Copy emergency scripts COPY --chown=dockhand:dockhand scripts/emergency/ ./scripts/ RUN chmod +x ./scripts/*.sh ./scripts/**/*.sh 2>/dev/null || true -# Create data directories with correct ownership +# Create data directories RUN mkdir -p /home/dockhand/.dockhand/stacks /app/data \ && chown dockhand:dockhand /app/data /home/dockhand /home/dockhand/.dockhand /home/dockhand/.dockhand/stacks @@ -189,4 +161,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD curl -f http://localhost:3000/ || exit 1 ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] -CMD ["bun", "run", "./build/index.js"] +CMD ["node", "/app/server.js"] diff --git a/bunfig.toml b/bunfig.toml index 51bc0ff..34ac71c 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -7,3 +7,7 @@ exact = true [run] # Enable source maps for better error messages sourcemap = "external" + +[test] +# Disable auth before any integration test runs +preload = ["./tests/helpers/preload.ts"] diff --git a/collector/go.mod b/collector/go.mod new file mode 100644 index 0000000..7832d3a --- /dev/null +++ b/collector/go.mod @@ -0,0 +1,3 @@ +module github.com/Finsys/dockhand/collector + +go 1.24 diff --git a/collector/main.go b/collector/main.go new file mode 100644 index 0000000..643ab38 --- /dev/null +++ b/collector/main.go @@ -0,0 +1,940 @@ +// Collection worker for Dockhand. +// +// A lightweight Go binary that handles background Docker API calls for +// metrics collection, event streaming, and disk usage checks. +// Communicates with the Node.js parent process via JSON lines on +// stdin (commands) and stdout (results). +package main + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "math" + "net" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" +) + +// --------------------------------------------------------------------------- +// IPC message types +// --------------------------------------------------------------------------- + +// Inbound (stdin) messages from Node.js parent. +type InMessage struct { + Type string `json:"type"` + EnvID int `json:"envId,omitempty"` + Name string `json:"name,omitempty"` + Config *EnvConfig `json:"config,omitempty"` + ConnectionType string `json:"connectionType,omitempty"` + HawserToken string `json:"hawserToken,omitempty"` + IntervalMs int `json:"intervalMs,omitempty"` + Mode string `json:"mode,omitempty"` + PollIntervalMs int `json:"pollIntervalMs,omitempty"` +} + +type EnvConfig struct { + Type string `json:"type"` // "socket", "http", "https" + SocketPath string `json:"socketPath,omitempty"` + Host string `json:"host,omitempty"` + Port int `json:"port,omitempty"` + CA string `json:"ca,omitempty"` + Cert string `json:"cert,omitempty"` + Key string `json:"key,omitempty"` + SkipVerify bool `json:"skipVerify,omitempty"` +} + +// Outbound (stdout) messages to Node.js parent. +type OutMessage struct { + Type string `json:"type"` + EnvID int `json:"envId,omitempty"` + // Status + Online *bool `json:"online,omitempty"` + Error string `json:"error,omitempty"` + // Events + Event json.RawMessage `json:"event,omitempty"` + // Disk + Data json.RawMessage `json:"data,omitempty"` + Info json.RawMessage `json:"info,omitempty"` + // Metrics + CPU *float64 `json:"cpu,omitempty"` + MemPct *float64 `json:"memPercent,omitempty"` + MemUsed *int64 `json:"memUsed,omitempty"` + MemTotal *int64 `json:"memTotal,omitempty"` + CPUCount *int `json:"cpuCount,omitempty"` +} + +// --------------------------------------------------------------------------- +// Docker API response types (minimal, only what we need) +// --------------------------------------------------------------------------- + +type containerInfo struct { + ID string `json:"Id"` + State string `json:"State"` +} + +type containerStats struct { + CPUStats struct { + CPUUsage struct { + TotalUsage uint64 `json:"total_usage"` + } `json:"cpu_usage"` + SystemCPUUsage uint64 `json:"system_cpu_usage"` + OnlineCPUs int `json:"online_cpus"` + } `json:"cpu_stats"` + PrecpuStats struct { + CPUUsage struct { + TotalUsage uint64 `json:"total_usage"` + } `json:"cpu_usage"` + SystemCPUUsage uint64 `json:"system_cpu_usage"` + } `json:"precpu_stats"` + MemoryStats struct { + Usage uint64 `json:"usage"` + Stats struct { + InactiveFile uint64 `json:"inactive_file"` + TotalInactiveFile uint64 `json:"total_inactive_file"` + } `json:"stats"` + } `json:"memory_stats"` +} + +type dockerInfo struct { + MemTotal int64 `json:"MemTotal"` + NCPU int `json:"NCPU"` +} + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const statsConcurrency = 8 // Max parallel stats calls per environment + +// --------------------------------------------------------------------------- +// Environment manager +// --------------------------------------------------------------------------- + +type environment struct { + id int + name string + connectionType string + hawserToken string + client *http.Client + streamClient *http.Client + transport *http.Transport + streamTransport *http.Transport + baseURL string + cancel context.CancelFunc + ctx context.Context + online bool + statusReported bool // true after first env_status message sent +} + +// closeTransports releases idle connections held by the environment's HTTP transports. +// Must be called when an environment is removed or reconfigured to prevent connection pool leaks. +func (e *environment) closeTransports() { + if e.transport != nil { + e.transport.CloseIdleConnections() + } + if e.streamTransport != nil { + e.streamTransport.CloseIdleConnections() + } +} + +type manager struct { + mu sync.Mutex + envs map[int]*environment + metricsInterval time.Duration + eventMode string // "stream" or "poll" + pollInterval time.Duration + diskInterval time.Duration + output *json.Encoder + outputMu sync.Mutex +} + +func newManager(output *json.Encoder) *manager { + return &manager{ + envs: make(map[int]*environment), + metricsInterval: 30 * time.Second, + eventMode: "stream", + pollInterval: 60 * time.Second, + diskInterval: 5 * time.Minute, + output: output, + } +} + +func (m *manager) send(msg OutMessage) { + m.outputMu.Lock() + defer m.outputMu.Unlock() + _ = m.output.Encode(msg) +} + +func boolPtr(v bool) *bool { return &v } +func float64Ptr(v float64) *float64 { return &v } +func int64Ptr(v int64) *int64 { return &v } +func intPtr(v int) *int { return &v } + +// drainAndClose discards a response body and closes it (for connection reuse). +func drainAndClose(resp *http.Response) { + if resp != nil && resp.Body != nil { + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + } +} + +// --------------------------------------------------------------------------- +// Docker HTTP client construction +// --------------------------------------------------------------------------- + +func buildClients(cfg *EnvConfig) (client *http.Client, streamClient *http.Client, tp *http.Transport, stp *http.Transport, baseURL string, err error) { + var transport *http.Transport + var streamTransport *http.Transport + + switch cfg.Type { + case "socket": + socketPath := cfg.SocketPath + if socketPath == "" { + socketPath = "/var/run/docker.sock" + } + dial := func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socketPath) + } + transport = &http.Transport{ + DialContext: dial, + MaxIdleConns: 16, + MaxIdleConnsPerHost: 16, + MaxConnsPerHost: 16, + IdleConnTimeout: 90 * time.Second, + } + streamTransport = &http.Transport{ + DialContext: dial, + MaxIdleConns: 4, + MaxIdleConnsPerHost: 4, + MaxConnsPerHost: 4, + IdleConnTimeout: 0, + } + baseURL = "http://localhost" + + case "http": + transport = &http.Transport{ + MaxIdleConns: 16, + MaxIdleConnsPerHost: 16, + MaxConnsPerHost: 16, + IdleConnTimeout: 90 * time.Second, + } + streamTransport = &http.Transport{ + MaxIdleConns: 4, + MaxIdleConnsPerHost: 4, + MaxConnsPerHost: 4, + IdleConnTimeout: 0, + } + baseURL = fmt.Sprintf("http://%s:%d", cfg.Host, cfg.Port) + + case "https": + tlsCfg, tlsErr := buildTLSConfig(cfg) + if tlsErr != nil { + return nil, nil, nil, nil, "", tlsErr + } + streamTLSCfg := tlsCfg.Clone() + + transport = &http.Transport{ + TLSClientConfig: tlsCfg, + MaxIdleConns: 16, + MaxIdleConnsPerHost: 16, + MaxConnsPerHost: 16, + IdleConnTimeout: 90 * time.Second, + } + streamTransport = &http.Transport{ + TLSClientConfig: streamTLSCfg, + MaxIdleConns: 4, + MaxIdleConnsPerHost: 4, + MaxConnsPerHost: 4, + IdleConnTimeout: 0, + } + baseURL = fmt.Sprintf("https://%s:%d", cfg.Host, cfg.Port) + + default: + return nil, nil, nil, nil, "", fmt.Errorf("unsupported connection type: %s", cfg.Type) + } + + client = &http.Client{Transport: transport, Timeout: 30 * time.Second} + streamClient = &http.Client{Transport: streamTransport, Timeout: 0} + return client, streamClient, transport, streamTransport, baseURL, nil +} + +func buildTLSConfig(cfg *EnvConfig) (*tls.Config, error) { + tlsCfg := &tls.Config{ + InsecureSkipVerify: cfg.SkipVerify, + ServerName: cfg.Host, // Explicit SNI for IP-based hosts + } + + if cfg.CA != "" { + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM([]byte(cfg.CA)) { + return nil, fmt.Errorf("failed to parse CA certificate") + } + tlsCfg.RootCAs = pool + } + + if cfg.Cert != "" && cfg.Key != "" { + cert, err := tls.X509KeyPair([]byte(cfg.Cert), []byte(cfg.Key)) + if err != nil { + return nil, fmt.Errorf("failed to parse client cert/key: %w", err) + } + tlsCfg.Certificates = []tls.Certificate{cert} + } + + return tlsCfg, nil +} + +// --------------------------------------------------------------------------- +// Docker API helpers +// --------------------------------------------------------------------------- + +func (e *environment) doRequest(ctx context.Context, method, path string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, e.baseURL+path, nil) + if err != nil { + return nil, err + } + if e.hawserToken != "" { + req.Header.Set("X-Hawser-Token", e.hawserToken) + } + return e.client.Do(req) +} + +func (e *environment) doStreamRequest(ctx context.Context, method, path string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, e.baseURL+path, nil) + if err != nil { + return nil, err + } + if e.hawserToken != "" { + req.Header.Set("X-Hawser-Token", e.hawserToken) + } + return e.streamClient.Do(req) +} + +func (e *environment) ping(ctx context.Context) bool { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + resp, err := e.doRequest(ctx, "GET", "/_ping") + if err != nil { + return false + } + drainAndClose(resp) + return resp.StatusCode == 200 +} + +// --------------------------------------------------------------------------- +// Metrics collection goroutine +// --------------------------------------------------------------------------- + +func (m *manager) runMetrics(env *environment) { + m.collectMetrics(env) + + ticker := time.NewTicker(m.metricsInterval) + defer ticker.Stop() + + for { + select { + case <-env.ctx.Done(): + return + case <-ticker.C: + m.mu.Lock() + interval := m.metricsInterval + m.mu.Unlock() + ticker.Reset(interval) + m.collectMetrics(env) + } + } +} + +func (m *manager) collectMetrics(env *environment) { + if !env.ping(env.ctx) { + if env.online || !env.statusReported { + env.online = false + env.statusReported = true + m.send(OutMessage{Type: "env_status", EnvID: env.id, Online: boolPtr(false), Error: "Docker not reachable"}) + } + return + } + + if !env.online || !env.statusReported { + env.online = true + env.statusReported = true + m.send(OutMessage{Type: "env_status", EnvID: env.id, Online: boolPtr(true)}) + } + + // List running containers + ctx, cancel := context.WithTimeout(env.ctx, 15*time.Second) + defer cancel() + + resp, err := env.doRequest(ctx, "GET", "/containers/json?all=false") + if err != nil { + m.send(OutMessage{Type: "error", EnvID: env.id, Error: fmt.Sprintf("list containers: %s", err)}) + return + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + io.Copy(io.Discard, resp.Body) + return + } + + var containers []containerInfo + if err := json.NewDecoder(resp.Body).Decode(&containers); err != nil { + return + } + + // Filter to running containers only + running := make([]containerInfo, 0, len(containers)) + for _, c := range containers { + if c.State == "running" { + running = append(running, c) + } + } + + // Collect stats per container (parallel, bounded concurrency) + type statsResult struct { + cpu float64 + mem uint64 + } + results := make([]statsResult, len(running)) + var wg sync.WaitGroup + sem := make(chan struct{}, statsConcurrency) + + for i, c := range running { + wg.Add(1) + go func(idx int, id string) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + sCtx, sCancel := context.WithTimeout(env.ctx, 10*time.Second) + defer sCancel() + + sResp, sErr := env.doRequest(sCtx, "GET", fmt.Sprintf("/containers/%s/stats?stream=false&one-shot=true", id)) + if sErr != nil { + return + } + defer sResp.Body.Close() + + if sResp.StatusCode/100 != 2 { + io.Copy(io.Discard, sResp.Body) + return + } + + var stats containerStats + if json.NewDecoder(sResp.Body).Decode(&stats) != nil { + return + } + + cpuDelta := float64(stats.CPUStats.CPUUsage.TotalUsage - stats.PrecpuStats.CPUUsage.TotalUsage) + sysDelta := float64(stats.CPUStats.SystemCPUUsage - stats.PrecpuStats.SystemCPUUsage) + cpuCount := stats.CPUStats.OnlineCPUs + if cpuCount == 0 { + cpuCount = 1 + } + + var cpuPct float64 + if sysDelta > 0 && cpuDelta > 0 { + cpuPct = (cpuDelta / sysDelta) * float64(cpuCount) * 100 + } + + memUsage := stats.MemoryStats.Usage + memCache := stats.MemoryStats.Stats.InactiveFile + if memCache == 0 { + memCache = stats.MemoryStats.Stats.TotalInactiveFile + } + actualMem := memUsage + if memCache > 0 && memCache < memUsage { + actualMem = memUsage - memCache + } + + results[idx] = statsResult{cpu: cpuPct, mem: actualMem} + }(i, c.ID) + } + wg.Wait() + + var totalCPU float64 + var totalMem uint64 + for _, r := range results { + totalCPU += r.cpu + totalMem += r.mem + } + + // Get docker info for MemTotal and NCPU + iCtx, iCancel := context.WithTimeout(env.ctx, 10*time.Second) + defer iCancel() + + var info dockerInfo + iResp, iErr := env.doRequest(iCtx, "GET", "/info") + if iErr == nil { + defer iResp.Body.Close() + if iResp.StatusCode/100 == 2 { + json.NewDecoder(iResp.Body).Decode(&info) + } else { + io.Copy(io.Discard, iResp.Body) + } + } + + memTotal := info.MemTotal + cpuCount := info.NCPU + if cpuCount == 0 { + cpuCount = 1 + } + + normalizedCPU := totalCPU / float64(cpuCount) + var memPct float64 + if memTotal > 0 { + memPct = (float64(totalMem) / float64(memTotal)) * 100 + } + + if !math.IsNaN(normalizedCPU) && !math.IsInf(normalizedCPU, 0) && memTotal > 0 { + m.send(OutMessage{ + Type: "metrics", + EnvID: env.id, + CPU: float64Ptr(normalizedCPU), + MemPct: float64Ptr(memPct), + MemUsed: int64Ptr(int64(totalMem)), + MemTotal: int64Ptr(memTotal), + CPUCount: intPtr(cpuCount), + }) + } +} + +// --------------------------------------------------------------------------- +// Event streaming goroutine +// --------------------------------------------------------------------------- + +func (m *manager) runEvents(env *environment) { + reconnectDelay := 5 * time.Second + maxReconnectDelay := 60 * time.Second + + // Reusable timer to avoid time.After leaks in select statements. + // Stopped and drained between uses to prevent firing stale timers. + delayTimer := time.NewTimer(0) + if !delayTimer.Stop() { + <-delayTimer.C + } + + waitOrCancel := func(d time.Duration) bool { + delayTimer.Reset(d) + select { + case <-env.ctx.Done(): + if !delayTimer.Stop() { + <-delayTimer.C + } + return false + case <-delayTimer.C: + return true + } + } + + for { + if env.ctx.Err() != nil { + return + } + + m.mu.Lock() + mode := m.eventMode + pollInterval := m.pollInterval + m.mu.Unlock() + + if mode == "poll" { + m.pollEvents(env) + if !waitOrCancel(pollInterval) { + return + } + continue + } + + // Stream mode + if !env.ping(env.ctx) { + if env.online || !env.statusReported { + env.online = false + env.statusReported = true + m.send(OutMessage{Type: "env_status", EnvID: env.id, Online: boolPtr(false), Error: "Docker not reachable"}) + } + if !waitOrCancel(reconnectDelay) { + return + } + reconnectDelay = minDuration(reconnectDelay*2, maxReconnectDelay) + continue + } + + if !env.online || !env.statusReported { + env.online = true + env.statusReported = true + m.send(OutMessage{Type: "env_status", EnvID: env.id, Online: boolPtr(true)}) + } + reconnectDelay = 5 * time.Second + + // Open event stream + resp, err := env.doStreamRequest(env.ctx, "GET", "/events?type=container") + if err != nil { + if env.ctx.Err() != nil { + return + } + env.online = false + m.send(OutMessage{Type: "env_status", EnvID: env.id, Online: boolPtr(false), Error: err.Error()}) + if !waitOrCancel(reconnectDelay) { + return + } + reconnectDelay = minDuration(reconnectDelay*2, maxReconnectDelay) + continue + } + + if resp.StatusCode/100 != 2 { + drainAndClose(resp) + if !waitOrCancel(reconnectDelay) { + return + } + reconnectDelay = minDuration(reconnectDelay*2, maxReconnectDelay) + continue + } + + // Read events line-by-line with a bounded buffer. + // Docker events are newline-delimited JSON; using bufio.Scanner + // avoids json.Decoder's unbounded internal buffer growth. + // + // Force-close the body on context cancellation so scanner.Scan() + // unblocks. Without this, the goroutine can leak if the transport's + // internal cancel watcher doesn't fire (Go runtime implementation detail). + bodyDone := make(chan struct{}) + go func() { + select { + case <-env.ctx.Done(): + resp.Body.Close() + case <-bodyDone: + } + }() + + eventScanner := bufio.NewScanner(resp.Body) + eventScanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) // 64KB initial, 1MB max + for eventScanner.Scan() { + if env.ctx.Err() != nil { + break + } + line := eventScanner.Bytes() + if len(line) == 0 { + continue + } + // Validate JSON and forward as raw message + if json.Valid(line) { + m.send(OutMessage{ + Type: "container_event", + EnvID: env.id, + Event: json.RawMessage(append([]byte(nil), line...)), + }) + } + } + close(bodyDone) + resp.Body.Close() + + if env.ctx.Err() != nil { + return + } + + // Stream ended — reconnect + if !waitOrCancel(reconnectDelay) { + return + } + reconnectDelay = minDuration(reconnectDelay*2, maxReconnectDelay) + } +} + +func (m *manager) pollEvents(env *environment) { + if !env.ping(env.ctx) { + if env.online || !env.statusReported { + env.online = false + env.statusReported = true + m.send(OutMessage{Type: "env_status", EnvID: env.id, Online: boolPtr(false), Error: "Docker not reachable"}) + } + return + } + + if !env.online || !env.statusReported { + env.online = true + env.statusReported = true + m.send(OutMessage{Type: "env_status", EnvID: env.id, Online: boolPtr(true)}) + } + + now := time.Now().Unix() + since := now - 30 + + ctx, cancel := context.WithTimeout(env.ctx, 15*time.Second) + defer cancel() + + resp, err := env.doRequest(ctx, "GET", fmt.Sprintf("/events?type=container&since=%d&until=%d", since, now)) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + io.Copy(io.Discard, resp.Body) + return + } + + pollScanner := bufio.NewScanner(resp.Body) + pollScanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for pollScanner.Scan() { + line := pollScanner.Bytes() + if len(line) == 0 { + continue + } + if json.Valid(line) { + m.send(OutMessage{ + Type: "container_event", + EnvID: env.id, + Event: json.RawMessage(append([]byte(nil), line...)), + }) + } + } +} + +// --------------------------------------------------------------------------- +// Disk usage check goroutine +// --------------------------------------------------------------------------- + +func (m *manager) runDiskChecks(env *environment) { + if os.Getenv("SKIP_DF_COLLECTION") != "" { + return + } + + initDelay := time.NewTimer(10 * time.Second) + select { + case <-env.ctx.Done(): + if !initDelay.Stop() { + <-initDelay.C + } + return + case <-initDelay.C: + } + m.checkDisk(env) + + ticker := time.NewTicker(m.diskInterval) + defer ticker.Stop() + + for { + select { + case <-env.ctx.Done(): + return + case <-ticker.C: + m.checkDisk(env) + } + } +} + +func (m *manager) checkDisk(env *environment) { + if !env.ping(env.ctx) { + return + } + + ctx, cancel := context.WithTimeout(env.ctx, 20*time.Second) + defer cancel() + + resp, err := env.doRequest(ctx, "GET", "/system/df") + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + io.Copy(io.Discard, resp.Body) + return + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024)) // 10MB cap + if err != nil { + return + } + + // Also fetch /info for DriverStatus (percentage-based disk warnings) + var infoBody json.RawMessage + iCtx, iCancel := context.WithTimeout(env.ctx, 10*time.Second) + defer iCancel() + iResp, iErr := env.doRequest(iCtx, "GET", "/info") + if iErr == nil { + if iResp.StatusCode/100 == 2 { + infoBody, _ = io.ReadAll(io.LimitReader(iResp.Body, 2*1024*1024)) // 2MB cap + } else { + io.Copy(io.Discard, iResp.Body) + } + iResp.Body.Close() + } + + m.send(OutMessage{ + Type: "disk_usage", + EnvID: env.id, + Data: json.RawMessage(body), + Info: infoBody, + }) +} + +// --------------------------------------------------------------------------- +// Environment lifecycle +// --------------------------------------------------------------------------- + +func (m *manager) configure(msg InMessage) { + m.mu.Lock() + defer m.mu.Unlock() + + if existing, ok := m.envs[msg.EnvID]; ok { + existing.cancel() + existing.closeTransports() + delete(m.envs, msg.EnvID) + } + + if msg.Config == nil { + return + } + + if msg.ConnectionType == "hawser-edge" { + return + } + + client, streamClient, transport, streamTransport, baseURL, err := buildClients(msg.Config) + if err != nil { + m.send(OutMessage{Type: "error", EnvID: msg.EnvID, Error: fmt.Sprintf("configure: %s", err)}) + return + } + + ctx, cancel := context.WithCancel(context.Background()) + env := &environment{ + id: msg.EnvID, + name: msg.Name, + connectionType: msg.ConnectionType, + hawserToken: msg.HawserToken, + client: client, + streamClient: streamClient, + transport: transport, + streamTransport: streamTransport, + baseURL: baseURL, + cancel: cancel, + ctx: ctx, + } + + m.envs[msg.EnvID] = env + + go m.runMetrics(env) + go m.runEvents(env) + go m.runDiskChecks(env) + + fmt.Fprintf(os.Stderr, "[collector] configured env %d (%s) type=%s base=%s\n", env.id, env.name, msg.ConnectionType, baseURL) +} + +func (m *manager) remove(envID int) { + m.mu.Lock() + defer m.mu.Unlock() + + if env, ok := m.envs[envID]; ok { + env.cancel() + env.closeTransports() + delete(m.envs, envID) + fmt.Fprintf(os.Stderr, "[collector] removed env %d\n", envID) + } +} + +func (m *manager) shutdown() { + m.mu.Lock() + defer m.mu.Unlock() + + for id, env := range m.envs { + env.cancel() + env.closeTransports() + delete(m.envs, id) + } + fmt.Fprintf(os.Stderr, "[collector] shutdown complete\n") +} + +func (m *manager) setMetricsInterval(ms int) { + m.mu.Lock() + defer m.mu.Unlock() + if ms > 0 { + m.metricsInterval = time.Duration(ms) * time.Millisecond + fmt.Fprintf(os.Stderr, "[collector] metrics interval set to %dms\n", ms) + } +} + +func (m *manager) setEventMode(mode string, pollMs int) { + m.mu.Lock() + defer m.mu.Unlock() + if mode != "" { + m.eventMode = mode + } + if pollMs > 0 { + m.pollInterval = time.Duration(pollMs) * time.Millisecond + } + fmt.Fprintf(os.Stderr, "[collector] event mode=%s pollInterval=%dms\n", m.eventMode, m.pollInterval/time.Millisecond) +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +func main() { + fmt.Fprintf(os.Stderr, "[collector] starting...\n") + + encoder := json.NewEncoder(os.Stdout) + mgr := newManager(encoder) + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) + + go func() { + <-sigCh + fmt.Fprintf(os.Stderr, "[collector] received signal, shutting down\n") + mgr.shutdown() + os.Exit(0) + }() + + mgr.send(OutMessage{Type: "ready"}) + + scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(make([]byte, 0, 64*1024), 10*1024*1024) // 64KB initial, grows to 10MB if needed + + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + + var msg InMessage + if err := json.Unmarshal(line, &msg); err != nil { + fmt.Fprintf(os.Stderr, "[collector] invalid message: %s\n", err) + continue + } + + switch msg.Type { + case "configure": + mgr.configure(msg) + case "remove": + mgr.remove(msg.EnvID) + case "set_metrics_interval": + mgr.setMetricsInterval(msg.IntervalMs) + case "set_event_mode": + mgr.setEventMode(msg.Mode, msg.PollIntervalMs) + case "shutdown": + mgr.shutdown() + os.Exit(0) + default: + fmt.Fprintf(os.Stderr, "[collector] unknown message type: %s\n", msg.Type) + } + } + + fmt.Fprintf(os.Stderr, "[collector] stdin closed, exiting\n") + mgr.shutdown() +} + +func minDuration(a, b time.Duration) time.Duration { + if a < b { + return a + } + return b +} diff --git a/package.json b/package.json index d569215..9533afc 100644 --- a/package.json +++ b/package.json @@ -1,17 +1,17 @@ { "name": "dockhand", "private": true, - "version": "1.0.18", + "version": "1.0.19", "type": "module", "scripts": { - "dev": "bunx --bun vite dev", - "prebuild": "bunx license-checker --json --production | jq 'to_entries | map({name: (.key | split(\"@\")[0:-1] | join(\"@\")), version: (.key | split(\"@\")[-1]), license: .value.licenses, repository: .value.repository}) | sort_by(.name)' > src/lib/data/dependencies.json.tmp && mv src/lib/data/dependencies.json.tmp src/lib/data/dependencies.json || true", - "build": "bunx --bun vite build && bun scripts/patch-build.ts && bun scripts/build-subprocesses.ts", - "start": "bun ./build/index.js", - "preview": "bun ./build/index.js", - "prepare": "bunx --bun svelte-kit sync || echo ''", - "check": "bunx --bun svelte-kit sync && bunx --bun svelte-check --tsconfig ./tsconfig.json", - "check:watch": "bunx --bun svelte-kit sync && bunx --bun svelte-check --tsconfig ./tsconfig.json --watch", + "dev": "npx vite dev", + "prebuild": "npx license-checker --json --production | jq 'to_entries | map({name: (.key | split(\"@\")[0:-1] | join(\"@\")), version: (.key | split(\"@\")[-1]), license: .value.licenses, repository: .value.repository}) | sort_by(.name)' > src/lib/data/dependencies.json.tmp && mv src/lib/data/dependencies.json.tmp src/lib/data/dependencies.json || true", + "build": "npx vite build", + "start": "node ./server.js", + "preview": "node ./build/index.js", + "prepare": "npx svelte-kit sync || echo ''", + "check": "npx svelte-kit sync && npx svelte-check --tsconfig ./tsconfig.json", + "check:watch": "npx svelte-kit sync && npx svelte-check --tsconfig ./tsconfig.json --watch", "test": "bun test", "test:smoke": "bun test tests/api-smoke.test.ts", "test:containers": "bun test tests/container-lifecycle.test.ts", @@ -49,8 +49,8 @@ "test:all": "bun test tests/", "test:quick": "bun test tests/api-smoke.test.ts tests/notifications.test.ts", "test:integration": "bun test tests/api-smoke.test.ts tests/crud-operations.test.ts tests/scheduling.test.ts tests/hawser-connection.test.ts", - "test:e2e": "bunx playwright test tests/e2e/", - "generate:legal": "bun scripts/generate-legal-pages.ts" + "test:e2e": "npx playwright test tests/e2e/", + "generate:legal": "node scripts/generate-legal-pages.ts" }, "dependencies": { "@codemirror/autocomplete": "6.20.0", @@ -71,11 +71,13 @@ "@codemirror/view": "6.39.11", "@lezer/highlight": "1.2.3", "@lucide/lab": "^0.1.2", + "argon2": "^0.41.1", + "better-sqlite3": "^11.7.0", "codemirror": "6.0.2", "croner": "9.1.0", "cronstrue": "3.9.0", + "devalue": "5.6.3", "drizzle-orm": "0.45.1", - "hash-wasm": "4.12.0", "js-yaml": "^4.1.1", "ldapts": "^8.1.3", "nodemailer": "^7.0.12", @@ -83,25 +85,29 @@ "postgres": "3.4.8", "qrcode": "^1.5.4", "svelte-dnd-action": "0.9.69", - "svelte-sonner": "1.0.7" + "svelte-sonner": "1.0.7", + "ws": "^8.18.0" }, "devDependencies": { "@internationalized/date": "^3.10.1", "@layerstack/tailwind": "^1.0.1", "@lucide/svelte": "^0.562.0", "@playwright/test": "1.57.0", + "@sveltejs/adapter-node": "^5.2.0", "@sveltejs/kit": "2.50.0", "@sveltejs/vite-plugin-svelte": "6.2.4", "@tailwindcss/vite": "^4.1.18", - "@types/bun": "1.3.6", + "@types/better-sqlite3": "^7.6.12", "@types/js-yaml": "^4.0.9", + "@types/node": "^22.10.0", "@types/nodemailer": "7.0.5", "@types/qrcode": "^1.5.6", + "@types/ws": "^8.5.13", "@xterm/addon-fit": "^0.11.0", "@xterm/addon-web-links": "^0.12.0", "@xterm/xterm": "^6.0.0", "autoprefixer": "^10.4.23", - "bits-ui": "^2.15.4", + "bits-ui": "2.15.4", "clsx": "^2.1.1", "cytoscape": "^3.33.1", "d3-scale": "^4.0.2", @@ -111,8 +117,7 @@ "lucide-svelte": "^0.562.0", "mode-watcher": "^1.1.0", "postcss": "^8.5.6", - "svelte": "5.47.1", - "svelte-adapter-bun": "1.0.1", + "svelte": "5.53.5", "svelte-check": "^4.3.5", "svelte-easy-crop": "^5.0.0", "svelte-virtual-scroll-list": "^1.3.0", diff --git a/scripts/build-subprocesses.ts b/scripts/build-subprocesses.ts deleted file mode 100644 index 35958e5..0000000 --- a/scripts/build-subprocesses.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Build subprocess scripts as standalone bundles for production. - * - * Subprocesses run via Bun.spawn and need all dependencies bundled - * since they can't access the SvelteKit build output's chunked modules. - */ - -const subprocesses = ['metrics-subprocess', 'event-subprocess']; - -console.log('[build-subprocesses] Bundling subprocess scripts...'); - -for (const name of subprocesses) { - const result = await Bun.build({ - entrypoints: [`./src/lib/server/subprocesses/${name}.ts`], - outdir: './build/subprocesses', - target: 'bun', - minify: false - }); - - if (!result.success) { - console.error(`[build-subprocesses] Failed to bundle ${name}:`); - for (const log of result.logs) { - console.error(log); - } - process.exit(1); - } - - console.log(`[build-subprocesses] Bundled ${name}.js`); -} - -console.log('[build-subprocesses] Done'); diff --git a/scripts/patch-build.ts b/scripts/patch-build.ts deleted file mode 100644 index bef946b..0000000 --- a/scripts/patch-build.ts +++ /dev/null @@ -1,690 +0,0 @@ -/** - * Post-build script to fix svelte-adapter-bun WebSocket issue - * The adapter calls server.websocket() which doesn't exist in SvelteKit. - * - * IMPORTANT: Terminal WebSocket logic is shared with vite.config.ts - * Core functions like resolveDockerTarget are defined in: - * src/lib/server/ws-terminal-shared.ts - * - * When updating WebSocket terminal handling, update the shared module - * and this file will use the same logic at build time. - */ - -import { join } from 'node:path'; - -const BUILD_DIR = join(import.meta.dir, '../build'); - -async function patchHandler() { - const handlerPath = join(BUILD_DIR, 'handler.js'); - const handlerFile = Bun.file(handlerPath); - - if (!await handlerFile.exists()) { - console.error('handler.js not found'); - process.exit(1); - } - - let content = await handlerFile.text(); - - // Replace broken server.websocket() call - content = content.replace( - 'const websocket = server.websocket();', - 'const websocket = null;' - ); - - // Add WebSocket upgrade detection before ssr handler - const ssrIndex = content.indexOf('var ssr = async (request, bunServer) => {'); - if (ssrIndex > -1) { - const upgradeCode = ` -var handleUpgrade = (request, bunServer) => { - const url = new URL(request.url); - const isUpgrade = request.headers.get('connection')?.toLowerCase().includes('upgrade') && - request.headers.get('upgrade')?.toLowerCase() === 'websocket'; - if (!isUpgrade) return null; - - // Handle terminal exec WebSocket - if (url.pathname.includes('/api/containers/') && url.pathname.includes('/exec')) { - const pathParts = url.pathname.split('/'); - const containerIdIndex = pathParts.indexOf('containers') + 1; - const containerId = pathParts[containerIdIndex]; - const shell = url.searchParams.get('shell') || '/bin/sh'; - const user = url.searchParams.get('user') || 'root'; - const envId = url.searchParams.get('envId') ? parseInt(url.searchParams.get('envId'), 10) : undefined; - if (bunServer.upgrade(request, { data: { type: 'terminal', containerId, shell, user, envId } })) { - return new Response(null, { status: 101 }); - } - } - - // Handle Hawser Edge WebSocket - if (url.pathname === '/api/hawser/connect') { - if (bunServer.upgrade(request, { data: { type: 'hawser' } })) { - return new Response(null, { status: 101 }); - } - } - - return null; -}; -`; - content = content.slice(0, ssrIndex) + upgradeCode + content.slice(ssrIndex); - } - - // Modify handler to check for upgrade first - content = content.replace( - 'return ssr(request, server2);', - 'const upgradeResponse = handleUpgrade(request, server2); if (upgradeResponse) return upgradeResponse; return ssr(request, server2);' - ); - - await Bun.write(handlerPath, content); - console.log('✓ Patched handler.js'); -} - -async function patchIndex() { - const indexPath = join(BUILD_DIR, 'index.js'); - const indexFile = Bun.file(indexPath); - - if (!await indexFile.exists()) { - console.error('index.js not found'); - process.exit(1); - } - - let content = await indexFile.text(); - - const wsHandler = ` -import { existsSync as _existsSync, readFileSync as _readFileSync } from 'fs'; -import { homedir as _homedir } from 'os'; -import { Database as _Database } from 'bun:sqlite'; -import { SQL as _SQL } from 'bun'; -import { join as _join } from 'path'; -import { createDecipheriv as _createDecipheriv } from 'node:crypto'; - -// Encryption/decryption for sensitive fields -const _ENCRYPTED_PREFIX = 'enc:v1:'; -const _IV_LENGTH = 12; -const _AUTH_TAG_LENGTH = 16; -let _encryptionKey = null; - -function _getEncryptionKey() { - if (_encryptionKey) return _encryptionKey; - const dataDir = process.env.DATA_DIR || _join(process.cwd(), 'data'); - const keyPath = _join(dataDir, '.encryption_key'); - const envKey = process.env.ENCRYPTION_KEY; - if (_existsSync(keyPath)) { - try { - _encryptionKey = _readFileSync(keyPath); - return _encryptionKey; - } catch {} - } - if (envKey) { - try { - _encryptionKey = Buffer.from(envKey, 'base64'); - return _encryptionKey; - } catch {} - } - return null; -} - -function _decrypt(value) { - if (!value || !value.startsWith(_ENCRYPTED_PREFIX)) return value; - const key = _getEncryptionKey(); - if (!key) { console.error('[WS] Cannot decrypt: no encryption key'); return value; } - try { - const payload = value.substring(_ENCRYPTED_PREFIX.length); - const combined = Buffer.from(payload, 'base64'); - if (combined.length < _IV_LENGTH + _AUTH_TAG_LENGTH + 1) return value; - const iv = combined.subarray(0, _IV_LENGTH); - const authTag = combined.subarray(_IV_LENGTH, _IV_LENGTH + _AUTH_TAG_LENGTH); - const ciphertext = combined.subarray(_IV_LENGTH + _AUTH_TAG_LENGTH); - const decipher = _createDecipheriv('aes-256-gcm', key, iv); - decipher.setAuthTag(authTag); - return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString('utf8'); - } catch (e) { console.error('[WS] Decryption failed:', e); return value; } -} - -// Database connection (supports both SQLite and PostgreSQL) -let _db = null; -let _isPostgres = false; -function _getDb() { - if (!_db) { - const dbUrl = process.env.DATABASE_URL; - if (dbUrl && (dbUrl.startsWith('postgres://') || dbUrl.startsWith('postgresql://'))) { - _db = new _SQL(dbUrl); - _isPostgres = true; - } else { - const _dbPath = process.env.DATA_DIR ? _join(process.env.DATA_DIR, 'db', 'dockhand.db') : _join(process.cwd(), 'data', 'db', 'dockhand.db'); - if (_existsSync(_dbPath)) { - _db = new _Database(_dbPath); - } - } - } - return _db; -} - -async function _getEnvironment(id) { - const db = _getDb(); - if (!db) return null; - let row; - if (_isPostgres) { - const result = await db.unsafe('SELECT * FROM environments WHERE id = $1', [id]); - row = result[0]; - } else { - row = db.prepare('SELECT * FROM environments WHERE id = ?').get(id); - } - return row ? { ...row, is_local: Boolean(row.is_local), connection_type: row.connection_type, hawser_token: row.hawser_token } : null; -} - -function detectDockerSocket() { - if (process.env.DOCKER_SOCKET && _existsSync(process.env.DOCKER_SOCKET)) return process.env.DOCKER_SOCKET; - if (process.env.DOCKER_HOST?.startsWith('unix://')) { - const p = process.env.DOCKER_HOST.replace('unix://', ''); - if (_existsSync(p)) return p; - } - for (const s of ['/var/run/docker.sock', _homedir() + '/.docker/run/docker.sock', _homedir() + '/.orbstack/run/docker.sock', '/run/docker.sock']) { - if (_existsSync(s)) return s; - } - return '/var/run/docker.sock'; -} -const dockerSocketPath = detectDockerSocket(); -console.log('Detected Docker socket at:', dockerSocketPath); - -const dockerStreams = new Map(); -let _wsConnCounter = 0; - -async function _getDockerTarget(envId) { - if (!envId) return { type: 'unix', socket: dockerSocketPath }; - const env = await _getEnvironment(envId); - if (!env) return { type: 'unix', socket: dockerSocketPath }; - // Check for socket connection type (local Unix socket) - if (env.is_local || env.connection_type === 'socket' || !env.connection_type) { - return { type: 'unix', socket: env.socket_path || dockerSocketPath }; - } - if (env.connection_type === 'hawser-edge') return { type: 'hawser-edge', environmentId: envId }; - // Build TLS config if using HTTPS - const protocol = env.protocol || 'http'; - const useTls = protocol === 'https'; - let tls = null; - if (useTls) { - tls = { - rejectUnauthorized: !env.tls_skip_verify, - ca: env.tls_ca || undefined, - cert: env.tls_cert || undefined, - // tls_key is encrypted - decrypt it - key: _decrypt(env.tls_key) || undefined - }; - } - // hawser_token is also encrypted - const hawserToken = env.connection_type === 'hawser-standard' && env.hawser_token - ? _decrypt(env.hawser_token) || undefined - : undefined; - return { - type: useTls ? 'tls' : 'tcp', - host: env.host, - port: env.port || 2375, - hawserToken, - tls - }; -} - -async function createExec(containerId, cmd, user, target) { - const headers = { 'Content-Type': 'application/json' }; - const fetchOpts = { - method: 'POST', - headers, - body: JSON.stringify({ AttachStdin: true, AttachStdout: true, AttachStderr: true, Tty: true, Cmd: cmd, User: user }) - }; - let url; - if (target.type === 'unix') { - url = 'http://localhost/containers/' + containerId + '/exec'; - fetchOpts.unix = target.socket; - } else { - const protocol = target.type === 'tls' ? 'https' : 'http'; - url = protocol + '://' + target.host + ':' + target.port + '/containers/' + containerId + '/exec'; - if (target.hawserToken) headers['X-Hawser-Token'] = target.hawserToken; - if (target.tls) { - fetchOpts.tls = { - sessionTimeout: 0, - servername: target.host, - rejectUnauthorized: target.tls.rejectUnauthorized - }; - if (target.tls.ca) fetchOpts.tls.ca = [target.tls.ca]; - if (target.tls.cert) fetchOpts.tls.cert = [target.tls.cert]; - if (target.tls.key) fetchOpts.tls.key = target.tls.key; - fetchOpts.keepalive = false; - } - } - const res = await fetch(url, fetchOpts); - if (!res.ok) throw new Error('Failed to create exec: ' + (await res.text())); - return res.json(); -} - -async function resizeExec(execId, cols, rows, target) { - try { - const fetchOpts = { method: 'POST' }; - let url; - if (target.type === 'unix') { - url = 'http://localhost/exec/' + execId + '/resize?h=' + rows + '&w=' + cols; - fetchOpts.unix = target.socket; - } else { - const protocol = target.type === 'tls' ? 'https' : 'http'; - url = protocol + '://' + target.host + ':' + target.port + '/exec/' + execId + '/resize?h=' + rows + '&w=' + cols; - if (target.hawserToken) fetchOpts.headers = { 'X-Hawser-Token': target.hawserToken }; - if (target.tls) { - fetchOpts.tls = { - sessionTimeout: 0, - servername: target.host, - rejectUnauthorized: target.tls.rejectUnauthorized - }; - if (target.tls.ca) fetchOpts.tls.ca = [target.tls.ca]; - if (target.tls.cert) fetchOpts.tls.cert = [target.tls.cert]; - if (target.tls.key) fetchOpts.tls.key = target.tls.key; - fetchOpts.keepalive = false; - } - } - await fetch(url, fetchOpts); - } catch {} -} - -// ============ Hawser Edge Support ============ -// Global edge connections map (shared with hawser.ts via globalThis) -if (!globalThis.__hawserEdgeConnections) globalThis.__hawserEdgeConnections = new Map(); -const _edgeConnections = globalThis.__hawserEdgeConnections; - -// Map WebSocket to environmentId for quick lookup -const _wsToEnvId = new Map(); - -// Edge exec sessions (execId -> frontend WebSocket) -const _edgeExecSessions = new Map(); - -// Validate Hawser token against database -async function _validateHawserToken(token) { - const db = _getDb(); - if (!db) return { valid: false }; - let tokens; - if (_isPostgres) { - tokens = await db.unsafe('SELECT * FROM hawser_tokens WHERE is_active = true'); - } else { - tokens = db.prepare('SELECT * FROM hawser_tokens WHERE is_active = 1').all(); - } - for (const t of tokens) { - try { - const isValid = await Bun.password.verify(token, t.token); - if (isValid) { - if (_isPostgres) { - await db.unsafe('UPDATE hawser_tokens SET last_used = NOW() WHERE id = $1', [t.id]); - } else { - db.prepare('UPDATE hawser_tokens SET last_used = datetime(\\'now\\') WHERE id = ?').run(t.id); - } - return { valid: true, environmentId: t.environment_id, tokenId: t.id }; - } - } catch {} - } - return { valid: false }; -} - -// Update environment status in database -async function _updateEnvStatus(envId, conn) { - const db = _getDb(); - if (!db) return; - try { - if (conn) { - if (_isPostgres) { - await db.unsafe('UPDATE environments SET hawser_last_seen = NOW(), hawser_agent_id = $1, hawser_agent_name = $2, hawser_version = $3, hawser_capabilities = $4 WHERE id = $5', - [conn.agentId, conn.agentName, conn.agentVersion, JSON.stringify(conn.capabilities || []), envId]); - } else { - db.prepare('UPDATE environments SET hawser_last_seen = datetime(\\'now\\'), hawser_agent_id = ?, hawser_agent_name = ?, hawser_version = ?, hawser_capabilities = ? WHERE id = ?') - .run(conn.agentId, conn.agentName, conn.agentVersion, JSON.stringify(conn.capabilities || []), envId); - } - } else { - if (_isPostgres) { - await db.unsafe('UPDATE environments SET hawser_last_seen = NOW() WHERE id = $1', [envId]); - } else { - db.prepare('UPDATE environments SET hawser_last_seen = datetime(\\'now\\') WHERE id = ?').run(envId); - } - } - } catch {} -} - -// Handle Hawser Edge protocol messages -async function _handleHawserMessage(ws, msg) { - if (msg.type === 'hello') { - console.log('[Hawser] Hello from agent:', msg.agentName, '(' + msg.agentId + ')'); - const validation = await _validateHawserToken(msg.token); - if (!validation.valid) { - console.log('[Hawser] Invalid token'); - ws.send(JSON.stringify({ type: 'error', error: 'Invalid token' })); - ws.close(); - return; - } - const envId = validation.environmentId; - const existing = _edgeConnections.get(envId); - if (existing) { - const pendingCount = existing.pendingRequests.size; - const streamCount = existing.pendingStreamRequests.size; - console.log('[Hawser] Replacing existing connection for env', envId, '- rejecting', pendingCount, 'pending requests and', streamCount, 'stream requests'); - // Reject all pending requests before closing - for (const [requestId, pending] of existing.pendingRequests) { - clearTimeout(pending.timeout); - pending.reject(new Error('Connection replaced by new agent')); - } - for (const [requestId, pending] of existing.pendingStreamRequests) { - pending.onEnd?.('Connection replaced by new agent'); - } - existing.pendingRequests.clear(); - existing.pendingStreamRequests.clear(); - existing.ws.close(1000, 'Replaced'); - _wsToEnvId.delete(existing.ws); - } - const conn = { - ws, environmentId: envId, agentId: msg.agentId, agentName: msg.agentName, - agentVersion: msg.version || 'unknown', dockerVersion: msg.dockerVersion || 'unknown', - hostname: msg.hostname || 'unknown', capabilities: msg.capabilities || [], - connectedAt: new Date(), lastHeartbeat: new Date(), - pendingRequests: new Map(), pendingStreamRequests: new Map(), - pingInterval: null - }; - _edgeConnections.set(envId, conn); - _wsToEnvId.set(ws, envId); - await _updateEnvStatus(envId, conn); - ws.send(JSON.stringify({ type: 'welcome', environmentId: envId, message: 'Connected to Dockhand' })); - // Start server-side ping interval to keep connection alive through Traefik/proxies (5s) - conn.pingInterval = setInterval(() => { - try { ws.send(JSON.stringify({ type: 'ping', timestamp: Date.now() })); } - catch { if (conn.pingInterval) { clearInterval(conn.pingInterval); conn.pingInterval = null; } } - }, 5000); - console.log('[Hawser] Agent', msg.agentName, 'connected for env', envId); - } else if (msg.type === 'ping') { - const envId = _wsToEnvId.get(ws); - if (envId) { const c = _edgeConnections.get(envId); if (c) c.lastHeartbeat = new Date(); } - ws.send(JSON.stringify({ type: 'pong', timestamp: Date.now() })); - } else if (msg.type === 'pong') { - const envId = _wsToEnvId.get(ws); - if (envId) { const c = _edgeConnections.get(envId); if (c) c.lastHeartbeat = new Date(); } - } else if (msg.type === 'response') { - const envId = _wsToEnvId.get(ws); - if (!envId) { - console.warn('[Hawser] Response from unknown WebSocket, requestId=' + msg.requestId); - return; - } - const conn = _edgeConnections.get(envId); - if (conn) { - const pending = conn.pendingRequests.get(msg.requestId); - if (pending) { - clearTimeout(pending.timeout); - conn.pendingRequests.delete(msg.requestId); - pending.resolve({ statusCode: msg.statusCode, headers: msg.headers || {}, body: msg.body || '', isBinary: msg.isBinary || false }); - } else { - console.warn('[Hawser] Response for unknown request ' + msg.requestId + ' on env ' + envId); - } - } - } else if (msg.type === 'stream') { - const envId = _wsToEnvId.get(ws); - if (!envId) { - console.warn('[Hawser] Stream data from unknown WebSocket, requestId=' + msg.requestId); - return; - } - const conn = _edgeConnections.get(envId); - if (conn?.pendingStreamRequests) { - const pending = conn.pendingStreamRequests.get(msg.requestId); - if (pending) { - pending.onData(msg.data, msg.stream); - } else { - console.warn('[Hawser] Stream data for unknown request ' + msg.requestId + ' on env ' + envId); - } - } - } else if (msg.type === 'stream_end') { - const envId = _wsToEnvId.get(ws); - if (!envId) { - console.warn('[Hawser] Stream end from unknown WebSocket, requestId=' + msg.requestId); - return; - } - const conn = _edgeConnections.get(envId); - if (conn?.pendingStreamRequests) { - const pending = conn.pendingStreamRequests.get(msg.requestId); - if (pending) { - conn.pendingStreamRequests.delete(msg.requestId); - pending.onEnd(msg.reason); - } else { - console.warn('[Hawser] Stream end for unknown request ' + msg.requestId + ' on env ' + envId); - } - } - } else if (msg.type === 'exec_ready') { - const session = _edgeExecSessions.get(msg.execId); - if (session?.ws?.readyState === 1) console.log('[Hawser] Exec ready:', msg.execId); - } else if (msg.type === 'exec_output') { - const session = _edgeExecSessions.get(msg.execId); - if (session?.ws?.readyState === 1) { - const data = Buffer.from(msg.data, 'base64').toString('utf-8'); - session.ws.send(JSON.stringify({ type: 'output', data })); - } - } else if (msg.type === 'exec_end') { - const session = _edgeExecSessions.get(msg.execId); - if (session) { - console.log('[Hawser] Exec ended:', msg.execId); - if (session.ws?.readyState === 1) { session.ws.send(JSON.stringify({ type: 'exit' })); session.ws.close(); } - _edgeExecSessions.delete(msg.execId); - } - } else if (msg.type === 'container_event') { - const envId = _wsToEnvId.get(ws); - if (envId && msg.event) { - // Call the global handler registered by hawser.ts - if (globalThis.__hawserHandleContainerEvent) { - globalThis.__hawserHandleContainerEvent(envId, msg.event).catch((err) => { - console.error('[Hawser] Error handling container event:', err); - }); - } - } - } else if (msg.type === 'metrics') { - // Metrics from agent - save to database for dashboard graphs - const envId = _wsToEnvId.get(ws); - if (envId && msg.metrics) { - if (globalThis.__hawserHandleMetrics) { - globalThis.__hawserHandleMetrics(envId, msg.metrics).catch((err) => { - console.error('[Hawser] Error saving metrics:', err); - }); - } - } - } -} - -// Expose send function for hawser.ts module -globalThis.__hawserSendMessage = (envId, message) => { - const conn = _edgeConnections.get(envId); - if (!conn?.ws) return false; - try { conn.ws.send(message); return true; } catch { return false; } -}; - -// ============ Combined WebSocket Handler ============ -const combinedWebsocket = { - async open(ws) { - const connType = ws.data?.type; - - // Hawser Edge connection - wait for hello message - if (connType === 'hawser') { - console.log('[Hawser] New connection pending authentication'); - return; - } - - // Terminal connection - const connId = 'ws-' + (++_wsConnCounter); - ws.data = ws.data || {}; - ws.data.connId = connId; - const { containerId, shell, user, envId } = ws.data; - if (!containerId) { ws.send(JSON.stringify({ type: 'error', message: 'No container ID' })); ws.close(); return; } - const target = await _getDockerTarget(envId); - console.log('[Terminal WS] Target:', JSON.stringify({ type: target.type, host: target.host, port: target.port, hasTls: !!target.tls, hasCa: !!target.tls?.ca, hasCert: !!target.tls?.cert, hasKey: !!target.tls?.key })); - - // Handle Hawser Edge terminal - if (target.type === 'hawser-edge') { - const conn = _edgeConnections.get(target.environmentId); - if (!conn) { ws.send(JSON.stringify({ type: 'error', message: 'Edge agent not connected' })); ws.close(); return; } - const execId = crypto.randomUUID(); - _edgeExecSessions.set(execId, { ws, execId, environmentId: target.environmentId }); - ws.data.edgeExecId = execId; - conn.ws.send(JSON.stringify({ type: 'exec_start', execId, containerId, cmd: shell || '/bin/sh', user: user || 'root', cols: 120, rows: 30 })); - return; - } - - try { - console.log('[Terminal WS] Creating exec for container:', containerId); - const exec = await createExec(containerId, [shell || '/bin/sh'], user || 'root', target); - console.log('[Terminal WS] Exec created:', exec?.Id); - const execId = exec.Id; - let dockerStream; - let headersStripped = false; - let isChunked = false; - const socketHandler = { - data(socket, data) { - if (ws.readyState === 1) { - let text = new TextDecoder().decode(data); - if (!headersStripped) { - if (text.toLowerCase().includes('transfer-encoding: chunked')) isChunked = true; - const i = text.indexOf('\\r\\n\\r\\n'); - if (i > -1) { text = text.slice(i + 4); headersStripped = true; } - else if (text.startsWith('HTTP/')) return; - } - if (isChunked && text) text = text.replace(/^[0-9a-fA-F]+\\r\\n/gm, '').replace(/\\r\\n$/g, ''); - if (text) ws.send(JSON.stringify({ type: 'output', data: text })); - } - }, - close() { if (ws.readyState === 1) { ws.send(JSON.stringify({ type: 'exit' })); ws.close(); } }, - error(socket, error) { - console.error('[Terminal WS] Socket error:', error?.message || error); - if (ws.readyState === 1) ws.send(JSON.stringify({ type: 'error', message: 'Connection error: ' + (error?.message || 'Unknown error') })); - }, - connectError(socket, error) { - console.error('[Terminal WS] Connect error:', error?.message || error); - if (ws.readyState === 1) { ws.send(JSON.stringify({ type: 'error', message: 'Failed to connect: ' + (error?.message || 'Unknown error') })); ws.close(); } - }, - open(socket) { - const body = JSON.stringify({ Detach: false, Tty: true }); - const tokenHeader = (target.type === 'tcp' || target.type === 'tls') && target.hawserToken ? 'X-Hawser-Token: ' + target.hawserToken + '\\r\\n' : ''; - // Use actual host for proper routing through reverse proxies like Caddy - const host = target.host || 'localhost'; - socket.write('POST /exec/' + execId + '/start HTTP/1.1\\r\\nHost: ' + host + '\\r\\nContent-Type: application/json\\r\\n' + tokenHeader + 'Connection: Upgrade\\r\\nUpgrade: tcp\\r\\nContent-Length: ' + body.length + '\\r\\n\\r\\n' + body); - } - }; - if (target.type === 'unix') { - dockerStream = await Bun.connect({ unix: target.socket, socket: socketHandler }); - } else { - const connectOpts = { hostname: target.host, port: target.port, socket: socketHandler }; - if (target.tls) { - connectOpts.tls = { - sessionTimeout: 0, - servername: target.host, - rejectUnauthorized: target.tls.rejectUnauthorized - }; - if (target.tls.ca) connectOpts.tls.ca = [target.tls.ca]; - if (target.tls.cert) connectOpts.tls.cert = [target.tls.cert]; - if (target.tls.key) connectOpts.tls.key = target.tls.key; - } - console.log('[Terminal WS] Connecting to:', connectOpts.hostname, connectOpts.port, 'TLS:', !!connectOpts.tls); - dockerStream = await Bun.connect(connectOpts); - console.log('[Terminal WS] Connected!'); - } - dockerStreams.set(connId, { stream: dockerStream, execId, target }); - } catch (e) { console.error('[Terminal WS] Error:', e); ws.send(JSON.stringify({ type: 'error', message: e.message })); ws.close(); } - }, - async message(ws, message) { - const connType = ws.data?.type; - - // Hawser Edge message - if (connType === 'hawser') { - try { - let msgStr = typeof message === 'string' ? message : message instanceof ArrayBuffer ? new TextDecoder().decode(message) : Buffer.isBuffer(message) ? message.toString('utf-8') : new TextDecoder().decode(new Uint8Array(message)); - const msg = JSON.parse(msgStr); - await _handleHawserMessage(ws, msg); - } catch (e) { - console.error('[Hawser] Error:', e.message); - ws.send(JSON.stringify({ type: 'error', error: e.message })); - } - return; - } - - // Edge exec session input - const edgeExecId = ws.data?.edgeExecId; - if (edgeExecId) { - const session = _edgeExecSessions.get(edgeExecId); - if (session) { - const conn = _edgeConnections.get(session.environmentId); - if (conn) { - try { - const msg = JSON.parse(message.toString()); - if (msg.type === 'input') conn.ws.send(JSON.stringify({ type: 'exec_input', execId: edgeExecId, data: Buffer.from(msg.data).toString('base64') })); - else if (msg.type === 'resize') conn.ws.send(JSON.stringify({ type: 'exec_resize', execId: edgeExecId, cols: msg.cols, rows: msg.rows })); - } catch {} - } - } - return; - } - - // Terminal message - const connId = ws.data?.connId; - if (!connId) return; - const d = dockerStreams.get(connId); - if (!d) return; - try { - const msg = JSON.parse(message.toString()); - if (msg.type === 'input' && d.stream) d.stream.write(msg.data); - else if (msg.type === 'resize' && d.execId) resizeExec(d.execId, msg.cols, msg.rows, d.target); - } catch { if (d.stream) d.stream.write(message); } - }, - close(ws) { - const connType = ws.data?.type; - - // Hawser Edge disconnection - if (connType === 'hawser') { - const envId = _wsToEnvId.get(ws); - if (envId) { - const conn = _edgeConnections.get(envId); - if (conn) { - console.log('[Hawser] Agent disconnected:', conn.agentId); - // Clear server-side ping interval - if (conn.pingInterval) { clearInterval(conn.pingInterval); conn.pingInterval = null; } - for (const [, p] of conn.pendingRequests) { clearTimeout(p.timeout); p.reject(new Error('Connection closed')); } - for (const [, p] of conn.pendingStreamRequests) { p.onEnd('Connection closed'); } - _edgeConnections.delete(envId); - _updateEnvStatus(envId, null); - } - _wsToEnvId.delete(ws); - } - return; - } - - // Edge exec session close - const edgeExecId = ws.data?.edgeExecId; - if (edgeExecId) { - const session = _edgeExecSessions.get(edgeExecId); - if (session) { - const conn = _edgeConnections.get(session.environmentId); - if (conn) conn.ws.send(JSON.stringify({ type: 'exec_end', execId: edgeExecId, reason: 'user_closed' })); - _edgeExecSessions.delete(edgeExecId); - } - return; - } - - // Terminal close - const connId = ws.data?.connId; - if (!connId) return; - const d = dockerStreams.get(connId); - if (d?.stream) d.stream.end(); - dockerStreams.delete(connId); - } -}; -`; - - const insertPoint = content.indexOf('var path = env('); - if (insertPoint > -1) { - content = content.slice(0, insertPoint) + wsHandler + content.slice(insertPoint); - } - - content = content.replace( - 'var { fetch: handlerFetch, websocket } = getHandler();', - 'var { fetch: handlerFetch, websocket: _ } = getHandler(); var websocket = combinedWebsocket;' - ); - - await Bun.write(indexPath, content); - console.log('✓ Patched index.js'); -} - -console.log('Patching build...'); -await patchHandler(); -await patchIndex(); -console.log('✓ Done'); diff --git a/src/hooks.server.ts b/src/hooks.server.ts index d5dfbaa..1d0ff2b 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -9,10 +9,74 @@ import { initCryptoFallback } from '$lib/server/crypto-fallback'; import { detectHostDataDir } from '$lib/server/host-path'; import { listContainers, removeContainer } from '$lib/server/docker'; import { migrateCredentials } from '$lib/server/encryption'; +import { gzipSync } from 'node:zlib'; import { rmSync, readdirSync, existsSync } from 'fs'; import { join } from 'path'; import type { HandleServerError, Handle } from '@sveltejs/kit'; import { redirect } from '@sveltejs/kit'; +import { startRssTracker, stopRssTracker, rssBeforeOp, rssAfterOp } from '$lib/server/rss-tracker'; + +// Content types worth compressing +const COMPRESSIBLE_TYPES = [ + 'application/json', + 'text/html', + 'text/plain', + 'text/css', + 'application/javascript', + 'text/javascript', + 'application/xml', + 'text/xml', + 'image/svg+xml' +]; + +// Minimum response size to bother compressing (1KB) +const MIN_COMPRESS_SIZE = 1024; + +function shouldCompress(request: Request, response: Response): boolean { + const acceptEncoding = request.headers.get('accept-encoding') || ''; + if (!acceptEncoding.includes('gzip')) return false; + + if (response.headers.has('content-encoding')) return false; + + const contentType = response.headers.get('content-type') || ''; + if (contentType.includes('text/event-stream')) return false; + if (contentType.includes('octet-stream')) return false; + if (contentType.startsWith('image/') && !contentType.includes('svg')) return false; + + const isCompressible = COMPRESSIBLE_TYPES.some(type => contentType.includes(type)); + if (!isCompressible) return false; + + const contentLength = response.headers.get('content-length'); + if (contentLength && parseInt(contentLength) < MIN_COMPRESS_SIZE) return false; + + return true; +} + +async function compressResponse(request: Request, response: Response): Promise { + if (!shouldCompress(request, response)) return response; + + const body = await response.arrayBuffer(); + if (body.byteLength < MIN_COMPRESS_SIZE) return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers + }); + + const gzipBefore = rssBeforeOp(); + const compressed = gzipSync(new Uint8Array(body)); + rssAfterOp('gzip', gzipBefore); + + const headers = new Headers(response.headers); + headers.set('content-encoding', 'gzip'); + headers.set('vary', 'Accept-Encoding'); + headers.delete('content-length'); + + return new Response(compressed, { + status: response.status, + statusText: response.statusText, + headers + }); +} // Cleanup orphaned scanner version containers from previous runs async function cleanupOrphanedScannerContainers() { @@ -98,11 +162,12 @@ if (!initialized) { cleanupOrphanedScannerContainers().catch(err => { console.error('Failed to cleanup orphaned scanner containers:', err); }); - // Start background subprocesses for metrics and event collection (isolated processes) + // Start background subprocesses for metrics and event collection (worker thread) startSubprocesses().catch(err => { console.error('Failed to start background subprocesses:', err); }); startScheduler(); // Start unified scheduler for auto-updates and git syncs (async) + startRssTracker(); // Start RSS memory tracking (no-op unless MEMORY_MONITOR=true) // Check license expiry on startup and then daily (with HMR guard) checkLicenseExpiry().catch(err => { @@ -119,6 +184,7 @@ if (!initialized) { // Graceful shutdown handling const shutdown = async () => { console.log('[Server] Shutting down...'); + stopRssTracker(); await stopSubprocesses(); process.exit(0); }; @@ -175,55 +241,57 @@ export const handle: Handle = async ({ event, resolve }) => { return resolve(event); } - // WebSocket upgrade for terminal connections is handled by the build patch (scripts/patch-build.ts) - // This is necessary because svelte-adapter-bun expects server.websocket() which doesn't exist in SvelteKit + const httpBefore = rssBeforeOp(); + try { + // Check if auth is enabled + const authEnabled = await isAuthEnabled(); - // Check if auth is enabled - const authEnabled = await isAuthEnabled(); - - // If auth is disabled, allow everything (app works as before) - if (!authEnabled) { - event.locals.user = null; - event.locals.authEnabled = false; - return resolve(event); - } - - // Auth is enabled - check session - const user = await validateSession(event.cookies); - event.locals.user = user; - event.locals.authEnabled = true; - - // Public paths don't require authentication - if (isPublicPath(event.url.pathname)) { - return resolve(event); - } - - // If not authenticated - if (!user) { - // Special case: allow user creation when auth is enabled but no admin exists yet - // This enables the first admin user to be created during initial setup - const noAdminSetupMode = !(await hasAdminUser()); - if (noAdminSetupMode && event.url.pathname === '/api/users' && event.request.method === 'POST') { - return resolve(event); + // If auth is disabled, allow everything (app works as before) + if (!authEnabled) { + event.locals.user = null; + event.locals.authEnabled = false; + return compressResponse(event.request, await resolve(event)); } - // API routes return 401 - if (event.url.pathname.startsWith('/api/')) { - return new Response( - JSON.stringify({ error: 'Unauthorized', message: 'Authentication required' }), - { - status: 401, - headers: { 'Content-Type': 'application/json' } - } - ); + // Auth is enabled - check session + const user = await validateSession(event.cookies); + event.locals.user = user; + event.locals.authEnabled = true; + + // Public paths don't require authentication + if (isPublicPath(event.url.pathname)) { + return compressResponse(event.request, await resolve(event)); } - // UI routes redirect to login - const redirectUrl = encodeURIComponent(event.url.pathname + event.url.search); - redirect(307, `/login?redirect=${redirectUrl}`); - } + // If not authenticated + if (!user) { + // Special case: allow user creation when auth is enabled but no admin exists yet + // This enables the first admin user to be created during initial setup + const noAdminSetupMode = !(await hasAdminUser()); + if (noAdminSetupMode && event.url.pathname === '/api/users' && event.request.method === 'POST') { + return compressResponse(event.request, await resolve(event)); + } - return resolve(event); + // API routes return 401 + if (event.url.pathname.startsWith('/api/')) { + return new Response( + JSON.stringify({ error: 'Unauthorized', message: 'Authentication required' }), + { + status: 401, + headers: { 'Content-Type': 'application/json' } + } + ); + } + + // UI routes redirect to login + const redirectUrl = encodeURIComponent(event.url.pathname + event.url.search); + redirect(307, `/login?redirect=${redirectUrl}`); + } + + return compressResponse(event.request, await resolve(event)); + } finally { + rssAfterOp('http', httpBefore); + } }; export const handleError: HandleServerError = ({ error, event }) => { diff --git a/src/lib/components/BatchOperationModal.svelte b/src/lib/components/BatchOperationModal.svelte index 304b0a9..67b66db 100644 --- a/src/lib/components/BatchOperationModal.svelte +++ b/src/lib/components/BatchOperationModal.svelte @@ -70,7 +70,7 @@ let successCount = $state(0); let failCount = $state(0); let cancelledCount = $state(0); - let abortController: AbortController | null = null; + let cancelled = false; // Progress calculation const progress = $derived(() => { @@ -88,9 +88,7 @@ // Cleanup on destroy onDestroy(() => { - if (abortController) { - abortController.abort(); - } + cancelled = true; }); async function startOperation() { @@ -106,20 +104,13 @@ successCount = 0; failCount = 0; cancelledCount = 0; - - abortController = new AbortController(); + cancelled = false; try { const response = await fetch(`/api/batch${envId ? `?env=${envId}` : ''}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - operation, - entityType, - items, - options - }), - signal: abortController.signal + body: JSON.stringify({ operation, entityType, items, options }) }); if (!response.ok) { @@ -127,52 +118,44 @@ throw new Error(error.error || 'Request failed'); } - if (!response.body) { - throw new Error('No response body'); - } + const data = await response.json(); + const { jobId } = data; - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; + // Poll job for progress events + let cursor = 0; + while (!cancelled) { + const jobRes = await fetch(`/api/jobs/${jobId}`); + if (!jobRes.ok) break; + const job = await jobRes.json(); - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split('\n\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - if (line.startsWith('data: ')) { - try { - const event: BatchEvent = JSON.parse(line.slice(6)); - handleEvent(event); - } catch { - // Ignore parse errors - } - } + // Process new lines since last poll + const newLines = job.lines.slice(cursor); + cursor = job.lines.length; + for (const line of newLines) { + handleEvent(line.data as BatchEvent); } + + if (job.status !== 'running') break; + await new Promise((r) => setTimeout(r, 500)); } - } catch (error: any) { - if (error.name === 'AbortError') { - // User cancelled - mark remaining as cancelled - let cancelled = 0; + + if (cancelled) { + // Mark remaining items as cancelled + let cancelCount = 0; itemStates = itemStates.map(item => { if (item.status === 'pending' || item.status === 'processing') { - cancelled++; + cancelCount++; return { ...item, status: 'cancelled' as ItemStatus }; } return item; }); - cancelledCount = cancelled; - } else { - console.error('Batch operation error:', error); + cancelledCount = cancelCount; } + } catch (error: any) { + console.error('Batch operation error:', error); } finally { isRunning = false; isComplete = true; - abortController = null; } } @@ -195,9 +178,7 @@ } function handleCancel() { - if (abortController) { - abortController.abort(); - } + cancelled = true; } function handleClose() { diff --git a/src/lib/components/PullTab.svelte b/src/lib/components/PullTab.svelte index f9a0227..41ad9ad 100644 --- a/src/lib/components/PullTab.svelte +++ b/src/lib/components/PullTab.svelte @@ -8,6 +8,7 @@ import { CheckCircle2, XCircle, Loader2, AlertCircle, Terminal, Sun, Moon, Download } from 'lucide-svelte'; import { onMount } from 'svelte'; import { appendEnvParam } from '$lib/stores/environment'; + import { watchJob } from '$lib/utils/sse-fetch'; interface LayerProgress { id: string; @@ -168,33 +169,10 @@ throw new Error('Failed to start pull'); } - const reader = response.body?.getReader(); - if (!reader) { - throw new Error('No response body'); - } - - const decoder = new TextDecoder(); - let buffer = ''; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - if (!line.trim() || !line.startsWith('data: ')) continue; - - try { - const data = JSON.parse(line.slice(6)); - handlePullProgress(data); - } catch (e) { - // Ignore parse errors - } - } - } + const { jobId } = await response.json(); + await watchJob(jobId, (line) => { + handlePullProgress(line.data as any); + }); if (status === 'pulling') { duration = Date.now() - startTime; diff --git a/src/lib/components/PushTab.svelte b/src/lib/components/PushTab.svelte index edce295..fe42688 100644 --- a/src/lib/components/PushTab.svelte +++ b/src/lib/components/PushTab.svelte @@ -2,6 +2,7 @@ import { tick, onMount } from 'svelte'; import { CheckCircle2, XCircle, Loader2, AlertCircle, Terminal, Sun, Moon, Upload } from 'lucide-svelte'; import { appendEnvParam } from '$lib/stores/environment'; + import { watchJob } from '$lib/utils/sse-fetch'; type PushStatus = 'idle' | 'pushing' | 'complete' | 'error'; @@ -144,39 +145,12 @@ return; } - // Handle SSE stream - const reader = pushResponse.body?.getReader(); - if (!reader) { - errorMessage = 'No response body'; - status = 'error'; - onError?.(errorMessage); - return; - } + const { jobId } = await pushResponse.json(); + await watchJob(jobId, (line) => { + handlePushProgress(line.data as any); + }); - const decoder = new TextDecoder(); - let buffer = ''; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - if (line.startsWith('data: ')) { - try { - const data = JSON.parse(line.slice(6)); - handlePushProgress(data); - } catch (e) { - // Ignore parse errors - } - } - } - } - - // If stream ended without complete/error status + // If job ended without an explicit complete/error event if (status === 'pushing') { status = 'complete'; statusMessage = 'Image pushed successfully!'; diff --git a/src/lib/components/ScanTab.svelte b/src/lib/components/ScanTab.svelte index 764d655..c79af14 100644 --- a/src/lib/components/ScanTab.svelte +++ b/src/lib/components/ScanTab.svelte @@ -5,6 +5,7 @@ import { Loader2, AlertCircle, Terminal, Sun, Moon, ShieldCheck, ShieldAlert, ShieldX, Shield } from 'lucide-svelte'; import { onMount } from 'svelte'; import { appendEnvParam } from '$lib/stores/environment'; + import { watchJob } from '$lib/utils/sse-fetch'; import ScanResultsView from '../../routes/images/ScanResultsView.svelte'; export interface ScanResult { @@ -148,31 +149,10 @@ throw new Error(`HTTP ${response.status}: ${response.statusText}`); } - const reader = response.body?.getReader(); - if (!reader) throw new Error('No response body'); - - const decoder = new TextDecoder(); - let buffer = ''; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - if (line.startsWith('data: ')) { - try { - const data = JSON.parse(line.slice(6)); - handleScanProgress(data); - } catch (e) { - // Ignore parse errors - } - } - } - } + const { jobId } = await response.json(); + await watchJob(jobId, (line) => { + handleScanProgress(line.data as any); + }); // If stream ended without complete status if (status === 'scanning') { diff --git a/src/lib/components/ui/sidebar/context.svelte.ts b/src/lib/components/ui/sidebar/context.svelte.ts index 15248ad..8fa7bcc 100644 --- a/src/lib/components/ui/sidebar/context.svelte.ts +++ b/src/lib/components/ui/sidebar/context.svelte.ts @@ -57,6 +57,10 @@ class SidebarState { ? (this.openMobile = !this.openMobile) : this.setOpen(!this.open); }; + + destroy = () => { + this.#isMobile.destroy(); + }; } const SYMBOL_KEY = "scn-sidebar"; diff --git a/src/lib/config/grid-columns.ts b/src/lib/config/grid-columns.ts index cf13407..77785b7 100644 --- a/src/lib/config/grid-columns.ts +++ b/src/lib/config/grid-columns.ts @@ -14,7 +14,7 @@ export const containerColumns: ColumnConfig[] = [ { id: 'networkIO', label: 'Net I/O', width: 85, minWidth: 70, align: 'right' }, { id: 'diskIO', label: 'Disk I/O', width: 85, minWidth: 70, align: 'right' }, { id: 'ip', label: 'IP', sortable: true, sortField: 'ip', width: 100, minWidth: 80 }, - { id: 'ports', label: 'Ports', width: 120, minWidth: 60 }, + { id: 'ports', label: 'Ports', sortable: true, sortField: 'ports', width: 120, minWidth: 60 }, { id: 'autoUpdate', label: 'Auto-update', width: 95, minWidth: 70 }, { id: 'stack', label: 'Stack', sortable: true, sortField: 'stack', width: 100, minWidth: 60 }, { id: 'actions', label: '', fixed: 'end', width: 200, minWidth: 150, resizable: true } diff --git a/src/lib/data/changelog.json b/src/lib/data/changelog.json index 0890c86..8bdde78 100644 --- a/src/lib/data/changelog.json +++ b/src/lib/data/changelog.json @@ -1,4 +1,23 @@ [ + { + "version": "1.0.19", + "comingSoon": true, + "changes": [ + { "type": "feature", "text": "Inline logs panel on stacks page — view container logs without leaving the page" }, + { "type": "feature", "text": "Make ports column sortable in containers grid" }, + { "type": "feature", "text": "Structured auth logging with client IP (login/logout/MFA/OIDC events)" }, + { "type": "fix", "text": "Fix memory leak: TLS context accumulation for HTTPS environments (Bun)" }, + { "type": "fix", "text": "Fix security scanning on Docker with custom logging drivers (Loki, Fluentd, etc.)" }, + { "type": "fix", "text": "Fix grouped log viewer not auto-scrolling on new entries" }, + { "type": "fix", "text": "Fix container recreation error messages not surfacing actual Docker errors" }, + { "type": "fix", "text": "Fix LDAP group-to-role mapping" }, + { "type": "fix", "text": "Fix container file browser hiding old files" }, + { "type": "fix", "text": "Fix SSH key permission issues on NAS filesystems" }, + { "type": "fix", "text": "Fix binary file corruption when syncing stacks to Hawser agents" }, + { "type": "fix", "text": "Fix UI timeout issues for long running operations" } + ], + "imageTag": "fnsys/dockhand:v1.0.19" + }, { "version": "1.0.18", "date": "2026-02-16", diff --git a/src/lib/data/dependencies.json b/src/lib/data/dependencies.json index 53e8d93..6b95063 100644 --- a/src/lib/data/dependencies.json +++ b/src/lib/data/dependencies.json @@ -5,6 +5,12 @@ "license": "MIT", "repository": "https://github.com/codemirror/autocomplete" }, + { + "name": "@codemirror/commands", + "version": "6.10.0", + "license": "MIT", + "repository": "https://github.com/codemirror/commands" + }, { "name": "@codemirror/commands", "version": "6.10.1", @@ -215,12 +221,24 @@ "license": "MIT", "repository": "https://github.com/paulmillr/noble-hashes" }, + { + "name": "@phc/format", + "version": "1.0.0", + "license": "MIT", + "repository": "https://github.com/simonepri/phc-format" + }, { "name": "@sveltejs/acorn-typescript", "version": "1.0.8", "license": "MIT", "repository": "https://github.com/sveltejs/acorn-typescript" }, + { + "name": "@types/better-sqlite3", + "version": "7.6.13", + "license": "MIT", + "repository": "https://github.com/DefinitelyTyped/DefinitelyTyped" + }, { "name": "@types/estree", "version": "1.0.8", @@ -233,6 +251,12 @@ "license": "MIT", "repository": "https://github.com/DefinitelyTyped/DefinitelyTyped" }, + { + "name": "@types/trusted-types", + "version": "2.0.7", + "license": "MIT", + "repository": "https://github.com/DefinitelyTyped/DefinitelyTyped" + }, { "name": "acorn", "version": "8.15.0", @@ -251,6 +275,12 @@ "license": "MIT", "repository": "https://github.com/chalk/ansi-styles" }, + { + "name": "argon2", + "version": "0.41.1", + "license": "MIT", + "repository": "https://github.com/ranisalt/node-argon2" + }, { "name": "argparse", "version": "2.0.1", @@ -269,6 +299,36 @@ "license": "Apache-2.0", "repository": "https://github.com/A11yance/axobject-query" }, + { + "name": "base64-js", + "version": "1.5.1", + "license": "MIT", + "repository": "https://github.com/beatgammit/base64-js" + }, + { + "name": "better-sqlite3", + "version": "11.10.0", + "license": "MIT", + "repository": "https://github.com/WiseLibs/better-sqlite3" + }, + { + "name": "bindings", + "version": "1.5.0", + "license": "MIT", + "repository": "https://github.com/TooTallNate/node-bindings" + }, + { + "name": "bl", + "version": "4.1.0", + "license": "MIT", + "repository": "https://github.com/rvagg/bl" + }, + { + "name": "buffer", + "version": "5.7.1", + "license": "MIT", + "repository": "https://github.com/feross/buffer" + }, { "name": "bun-types", "version": "1.3.6", @@ -281,6 +341,12 @@ "license": "MIT", "repository": "https://github.com/sindresorhus/camelcase" }, + { + "name": "chownr", + "version": "1.1.4", + "license": "ISC", + "repository": "https://github.com/isaacs/chownr" + }, { "name": "cliui", "version": "6.0.0", @@ -335,9 +401,27 @@ "license": "MIT", "repository": "https://github.com/sindresorhus/decamelize" }, + { + "name": "decompress-response", + "version": "6.0.0", + "license": "MIT", + "repository": "https://github.com/sindresorhus/decompress-response" + }, + { + "name": "deep-extend", + "version": "0.6.0", + "license": "MIT", + "repository": "https://github.com/unclechu/node-deep-extend" + }, + { + "name": "detect-libc", + "version": "2.1.2", + "license": "Apache-2.0", + "repository": "https://github.com/lovell/detect-libc" + }, { "name": "devalue", - "version": "5.6.2", + "version": "5.6.3", "license": "MIT", "repository": "https://github.com/sveltejs/devalue" }, @@ -349,7 +433,7 @@ }, { "name": "dockhand", - "version": "1.0.3", + "version": "1.0.18", "license": "UNLICENSED", "repository": null }, @@ -365,6 +449,12 @@ "license": "MIT", "repository": "https://github.com/mathiasbynens/emoji-regex" }, + { + "name": "end-of-stream", + "version": "1.4.5", + "license": "MIT", + "repository": "https://github.com/mafintosh/end-of-stream" + }, { "name": "esm-env", "version": "1.2.2", @@ -373,16 +463,34 @@ }, { "name": "esrap", - "version": "2.2.1", + "version": "2.2.3", "license": "MIT", "repository": "https://github.com/sveltejs/esrap" }, + { + "name": "expand-template", + "version": "2.0.3", + "license": "(MIT OR WTFPL)", + "repository": "https://github.com/ralphtheninja/expand-template" + }, + { + "name": "file-uri-to-path", + "version": "1.0.0", + "license": "MIT", + "repository": "https://github.com/TooTallNate/file-uri-to-path" + }, { "name": "find-up", "version": "4.1.0", "license": "MIT", "repository": "https://github.com/sindresorhus/find-up" }, + { + "name": "fs-constants", + "version": "1.0.0", + "license": "MIT", + "repository": "https://github.com/mafintosh/fs-constants" + }, { "name": "get-caller-file", "version": "2.0.5", @@ -390,10 +498,28 @@ "repository": "https://github.com/stefanpenner/get-caller-file" }, { - "name": "hash-wasm", - "version": "4.12.0", + "name": "github-from-package", + "version": "0.0.0", "license": "MIT", - "repository": "https://github.com/Daninet/hash-wasm" + "repository": "https://github.com/substack/github-from-package" + }, + { + "name": "ieee754", + "version": "1.2.1", + "license": "BSD-3-Clause", + "repository": "https://github.com/feross/ieee754" + }, + { + "name": "inherits", + "version": "2.0.4", + "license": "ISC", + "repository": "https://github.com/isaacs/inherits" + }, + { + "name": "ini", + "version": "1.3.8", + "license": "ISC", + "repository": "https://github.com/isaacs/ini" }, { "name": "is-fullwidth-code-point", @@ -437,12 +563,60 @@ "license": "MIT", "repository": "https://github.com/Rich-Harris/magic-string" }, + { + "name": "mimic-response", + "version": "3.1.0", + "license": "MIT", + "repository": "https://github.com/sindresorhus/mimic-response" + }, + { + "name": "minimist", + "version": "1.2.8", + "license": "MIT", + "repository": "https://github.com/minimistjs/minimist" + }, + { + "name": "mkdirp-classic", + "version": "0.5.3", + "license": "MIT", + "repository": "https://github.com/mafintosh/mkdirp-classic" + }, + { + "name": "napi-build-utils", + "version": "2.0.0", + "license": "MIT", + "repository": "https://github.com/inspiredware/napi-build-utils" + }, + { + "name": "node-abi", + "version": "3.87.0", + "license": "MIT", + "repository": "https://github.com/electron/node-abi" + }, + { + "name": "node-addon-api", + "version": "8.5.0", + "license": "MIT", + "repository": "https://github.com/nodejs/node-addon-api" + }, + { + "name": "node-gyp-build", + "version": "4.8.4", + "license": "MIT", + "repository": "https://github.com/prebuild/node-gyp-build" + }, { "name": "nodemailer", "version": "7.0.12", "license": "MIT-0", "repository": "https://github.com/nodemailer/nodemailer" }, + { + "name": "once", + "version": "1.4.0", + "license": "ISC", + "repository": "https://github.com/isaacs/once" + }, { "name": "otpauth", "version": "9.4.1", @@ -485,6 +659,18 @@ "license": "Unlicense", "repository": "https://github.com/porsager/postgres" }, + { + "name": "prebuild-install", + "version": "7.1.3", + "license": "MIT", + "repository": "https://github.com/prebuild/prebuild-install" + }, + { + "name": "pump", + "version": "3.0.3", + "license": "MIT", + "repository": "https://github.com/mafintosh/pump" + }, { "name": "punycode", "version": "2.3.1", @@ -497,6 +683,18 @@ "license": "MIT", "repository": "https://github.com/soldair/node-qrcode" }, + { + "name": "rc", + "version": "1.2.8", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "repository": "https://github.com/dominictarr/rc" + }, + { + "name": "readable-stream", + "version": "3.6.2", + "license": "MIT", + "repository": "https://github.com/nodejs/readable-stream" + }, { "name": "require-directory", "version": "2.1.1", @@ -515,12 +713,36 @@ "license": "MIT", "repository": "https://github.com/svecosystem/runed" }, + { + "name": "safe-buffer", + "version": "5.2.1", + "license": "MIT", + "repository": "https://github.com/feross/safe-buffer" + }, + { + "name": "semver", + "version": "7.7.4", + "license": "ISC", + "repository": "https://github.com/npm/node-semver" + }, { "name": "set-blocking", "version": "2.0.0", "license": "ISC", "repository": "https://github.com/yargs/set-blocking" }, + { + "name": "simple-concat", + "version": "1.0.1", + "license": "MIT", + "repository": "https://github.com/feross/simple-concat" + }, + { + "name": "simple-get", + "version": "4.0.1", + "license": "MIT", + "repository": "https://github.com/feross/simple-get" + }, { "name": "strict-event-emitter-types", "version": "2.0.0", @@ -533,12 +755,24 @@ "license": "MIT", "repository": "https://github.com/sindresorhus/string-width" }, + { + "name": "string_decoder", + "version": "1.3.0", + "license": "MIT", + "repository": "https://github.com/nodejs/string_decoder" + }, { "name": "strip-ansi", "version": "6.0.1", "license": "MIT", "repository": "https://github.com/chalk/strip-ansi" }, + { + "name": "strip-json-comments", + "version": "2.0.1", + "license": "MIT", + "repository": "https://github.com/sindresorhus/strip-json-comments" + }, { "name": "style-mod", "version": "4.1.3", @@ -547,7 +781,7 @@ }, { "name": "svelte", - "version": "5.46.4", + "version": "5.53.1", "license": "MIT", "repository": "https://github.com/sveltejs/svelte" }, @@ -563,18 +797,42 @@ "license": "MIT", "repository": "https://github.com/wobsoriano/svelte-sonner" }, + { + "name": "tar-fs", + "version": "2.1.4", + "license": "MIT", + "repository": "https://github.com/mafintosh/tar-fs" + }, + { + "name": "tar-stream", + "version": "2.2.0", + "license": "MIT", + "repository": "https://github.com/mafintosh/tar-stream" + }, { "name": "tr46", "version": "6.0.0", "license": "MIT", "repository": "https://github.com/jsdom/tr46" }, + { + "name": "tunnel-agent", + "version": "0.6.0", + "license": "Apache-2.0", + "repository": "https://github.com/mikeal/tunnel-agent" + }, { "name": "undici-types", "version": "7.16.0", "license": "MIT", "repository": "https://github.com/nodejs/undici" }, + { + "name": "util-deprecate", + "version": "1.0.2", + "license": "MIT", + "repository": "https://github.com/TooTallNate/util-deprecate" + }, { "name": "w3c-keyname", "version": "2.2.8", @@ -605,6 +863,18 @@ "license": "MIT", "repository": "https://github.com/chalk/wrap-ansi" }, + { + "name": "wrappy", + "version": "1.0.2", + "license": "ISC", + "repository": "https://github.com/npm/wrappy" + }, + { + "name": "ws", + "version": "8.19.0", + "license": "MIT", + "repository": "https://github.com/websockets/ws" + }, { "name": "y18n", "version": "4.0.3", diff --git a/src/lib/hooks/is-mobile.svelte.ts b/src/lib/hooks/is-mobile.svelte.ts index a60c2c7..cf6788c 100644 --- a/src/lib/hooks/is-mobile.svelte.ts +++ b/src/lib/hooks/is-mobile.svelte.ts @@ -5,6 +5,9 @@ const DEFAULT_MOBILE_BREAKPOINT = 768; export class IsMobile { #breakpoint: number; #current = $state(false); + #handleResize: (() => void) | null = null; + #handleMediaChange: ((e: MediaQueryListEvent) => void) | null = null; + #mql: MediaQueryList | null = null; constructor(breakpoint: number = DEFAULT_MOBILE_BREAKPOINT) { this.#breakpoint = breakpoint; @@ -14,22 +17,31 @@ export class IsMobile { this.#current = window.innerWidth < this.#breakpoint; // Listen for resize events - const handleResize = () => { + this.#handleResize = () => { this.#current = window.innerWidth < this.#breakpoint; }; - window.addEventListener('resize', handleResize); + window.addEventListener('resize', this.#handleResize); // Also use matchMedia for more reliable detection - const mql = window.matchMedia(`(max-width: ${this.#breakpoint - 1}px)`); - const handleMediaChange = (e: MediaQueryListEvent) => { + this.#mql = window.matchMedia(`(max-width: ${this.#breakpoint - 1}px)`); + this.#handleMediaChange = (e: MediaQueryListEvent) => { this.#current = e.matches; }; - mql.addEventListener('change', handleMediaChange); + this.#mql.addEventListener('change', this.#handleMediaChange); } } get current() { return this.#current; } + + destroy() { + if (this.#handleResize) { + window.removeEventListener('resize', this.#handleResize); + } + if (this.#mql && this.#handleMediaChange) { + this.#mql.removeEventListener('change', this.#handleMediaChange); + } + } } diff --git a/src/lib/server/auth.ts b/src/lib/server/auth.ts index 3bcd12f..d78695c 100644 --- a/src/lib/server/auth.ts +++ b/src/lib/server/auth.ts @@ -2,16 +2,16 @@ * Core Authentication Module * * Security features: - * - Argon2id password hashing via Bun.password (memory-hard, timing-attack resistant) + * - Argon2id password hashing via argon2 (memory-hard, timing-attack resistant) * - Cryptographically secure 32-byte random session tokens * - HttpOnly cookies (prevents XSS from reading tokens) - * - Secure flag (HTTPS only in production) + * - Secure flag (protocol-aware: x-forwarded-proto or COOKIE_SECURE env var, default off) * - SameSite=Strict (CSRF protection) */ import os from 'node:os'; -import { secureRandomBytes, usingFallback } from './crypto-fallback'; -import { argon2id, argon2Verify } from 'hash-wasm'; +import { createHash } from 'node:crypto'; +import argon2 from 'argon2'; import type { Cookies } from '@sveltejs/kit'; import { getAuthSettings, @@ -43,6 +43,7 @@ import { } from './db'; import { Client as LdapClient } from 'ldapts'; import { isEnterprise } from './license'; +import { secureRandomBytes } from './crypto-fallback'; // Session cookie name const SESSION_COOKIE_NAME = 'dockhand_session'; @@ -98,42 +99,25 @@ export interface LoginResult { // Password Hashing (Argon2id) // ============================================ -// Argon2id parameters (matching Bun.password defaults) +// Argon2id parameters const ARGON2_MEMORY_COST = 65536; // 64 MB in kibibytes const ARGON2_TIME_COST = 3; // 3 iterations const ARGON2_PARALLELISM = 1; // Single-threaded const ARGON2_HASH_LENGTH = 32; // 256-bit output -const ARGON2_SALT_LENGTH = 16; // 128-bit salt /** * Hash a password using Argon2id * - * On modern kernels (>=3.17): Uses Bun's native password API (faster) - * On old kernels (<3.17): Uses hash-wasm (WASM-based, no getrandom dependency) - * - * Argon2id is the recommended variant - resistant to both side-channel and GPU attacks + * Uses the argon2 npm package (C binding) for native performance. + * Returns PHC format: $argon2id$v=19$m=65536,t=3,p=1$... */ export async function hashPassword(password: string): Promise { - // On old kernels, Bun.password.hash() crashes because it internally uses getrandom() - // Use hash-wasm as a fallback which is pure WASM and doesn't depend on the syscall - if (usingFallback()) { - const salt = secureRandomBytes(ARGON2_SALT_LENGTH); - return argon2id({ - password, - salt, - iterations: ARGON2_TIME_COST, - parallelism: ARGON2_PARALLELISM, - memorySize: ARGON2_MEMORY_COST, - hashLength: ARGON2_HASH_LENGTH, - outputType: 'encoded' // Returns PHC format: $argon2id$v=19$m=65536,t=3,p=1$... - }); - } - - // Modern kernels: use Bun's native implementation (faster) - return Bun.password.hash(password, { - algorithm: 'argon2id', + return argon2.hash(password, { + type: argon2.argon2id, memoryCost: ARGON2_MEMORY_COST, - timeCost: ARGON2_TIME_COST + timeCost: ARGON2_TIME_COST, + parallelism: ARGON2_PARALLELISM, + hashLength: ARGON2_HASH_LENGTH }); } @@ -141,18 +125,13 @@ export async function hashPassword(password: string): Promise { * Verify a password against a hash * Uses constant-time comparison internally * - * Both Bun.password and hash-wasm use the same PHC format, so hashes are compatible + * The argon2 npm package uses standard PHC format, compatible with existing hashes */ export async function verifyPassword(password: string, hash: string): Promise { try { - // On old kernels, use hash-wasm for verification - if (usingFallback()) { - return await argon2Verify({ password, hash }); - } - - // Modern kernels: use Bun's native implementation - return await Bun.password.verify(password, hash); - } catch { + return await argon2.verify(hash, password); + } catch (e) { + console.error('[Auth] argon2.verify() threw unexpectedly:', e); return false; } } @@ -169,14 +148,43 @@ function generateSessionToken(): string { return secureRandomBytes(32).toString('base64url'); } +/** + * Determine whether to set the Secure flag on the session cookie. + * + * Priority: + * 1. COOKIE_SECURE env var ('true' / 'false') — explicit override + * 2. x-forwarded-proto header — set by Traefik / Nginx / Caddy + * 3. false — default, matches v1.0.18 Bun behavior + * + * Defaulting to false (not NODE_ENV) is intentional: Dockhand is commonly + * run over plain HTTP in homelabs. Setting Secure unconditionally in production + * causes a login loop when there is no HTTPS reverse proxy, because browsers + * silently discard Secure cookies on HTTP connections. + * + * Users behind an HTTPS reverse proxy get Secure cookies automatically via + * x-forwarded-proto. Users who terminate TLS in the app itself can opt in + * with COOKIE_SECURE=true. + */ +function isSecureContext(request?: Request): boolean { + if (process.env.COOKIE_SECURE === 'false') return false; + if (process.env.COOKIE_SECURE === 'true') return true; + if (request) { + const proto = request.headers.get('x-forwarded-proto'); + if (proto === 'https') return true; + } + return false; +} + /** * Create a new session for a user * @param provider - Auth provider: 'local', or provider name like 'Keycloak', 'Azure AD', etc. + * @param request - Optional incoming request (used to detect HTTPS via x-forwarded-proto) */ export async function createUserSession( userId: number, provider: string, - cookies: Cookies + cookies: Cookies, + request?: Request ): Promise { // Clean up expired sessions periodically await deleteExpiredSessions(); @@ -198,7 +206,7 @@ export async function createUserSession( const session = await dbCreateSession(sessionId, userId, provider, expiresAt); // Set secure cookie - setSessionCookie(cookies, sessionId, sessionTimeout); + setSessionCookie(cookies, sessionId, sessionTimeout, request); // Update user's last login time await updateUser(userId, { lastLogin: new Date().toISOString() }); @@ -209,11 +217,11 @@ export async function createUserSession( /** * Set the session cookie with secure attributes */ -function setSessionCookie(cookies: Cookies, sessionId: string, maxAge: number): void { +function setSessionCookie(cookies: Cookies, sessionId: string, maxAge: number, request?: Request): void { cookies.set(SESSION_COOKIE_NAME, sessionId, { path: '/', httpOnly: true, // Prevents XSS attacks from reading cookie - secure: process.env.NODE_ENV === 'production', // HTTPS only in production + secure: isSecureContext(request), // Protocol-aware: checks x-forwarded-proto or NODE_ENV sameSite: 'strict', // CSRF protection maxAge: maxAge // Session timeout in seconds }); @@ -563,6 +571,19 @@ export async function authenticateLdap( return { success: false, error: 'Invalid username or password' }; } +/** + * Escape special characters in an LDAP filter value (RFC 4515). + * Prevents LDAP injection via wildcards or control characters. + */ +function escapeLdapFilterValue(value: string): string { + return value + .replace(/\\/g, '\\5c') + .replace(/\*/g, '\\2a') + .replace(/\(/g, '\\28') + .replace(/\)/g, '\\29') + .replace(/\0/g, '\\00'); +} + /** * Try authentication against a specific LDAP configuration */ @@ -585,8 +606,10 @@ async function tryLdapAuth( await client.bind(config.bindDn, config.bindPassword); } - // Search for the user - const filter = config.userFilter.replace('{{username}}', username); + // Escape the username before interpolating into the LDAP filter (RFC 4515) + // to prevent LDAP injection via wildcards or special characters. + const safeUsername = escapeLdapFilterValue(username); + const filter = config.userFilter.replace('{{username}}', safeUsername); const { searchEntries } = await client.search(config.baseDn, { scope: 'sub', filter: filter, @@ -599,9 +622,11 @@ async function tryLdapAuth( ] }); + // Use a single generic error for both "not found" and "wrong password" + // to avoid leaking whether a username exists via response content or timing. if (searchEntries.length === 0) { await client.unbind(); - return { success: false, error: 'User not found' }; + return { success: false, error: 'Invalid username or password' }; } const userEntry = searchEntries[0]; @@ -666,11 +691,11 @@ async function tryLdapAuth( if (adminRole) { const hasAdminRole = await userHasAdminRole(user.id); if (shouldBeAdmin && !hasAdminRole) { - // Assign Admin role await assignUserRole(user.id, adminRole.id, null); + } else if (!shouldBeAdmin && hasAdminRole && config.adminGroup) { + // Remove Admin role if user is no longer in admin group + await removeUserRole(user.id, adminRole.id); } - // Note: We don't remove Admin role if not in LDAP group anymore - // to prevent accidental lockouts (same behavior as before) } // Process role mappings (Enterprise feature) @@ -683,14 +708,29 @@ async function tryLdapAuth( const userExistingRoles = await getUserRoles(user.id); const existingRoleIds = new Set(userExistingRoles.map(r => r.roleId)); - for (const mapping of roleMappings) { - // Skip if user already has this role - if (existingRoleIds.has(mapping.roleId)) continue; + // All role IDs referenced in mappings (these are LDAP-managed) + const mappedRoleIds = new Set(roleMappings.map(m => m.roleId)); - // Check if user is a member of the LDAP group + // Determine which mapped roles user should have based on current group membership + const shouldHaveRoleIds = new Set(); + for (const mapping of roleMappings) { const isInGroup = await checkLdapGroupMembership(config, userDn, mapping.groupDn); if (isInGroup) { - await assignUserRole(user.id, mapping.roleId, undefined); + shouldHaveRoleIds.add(mapping.roleId); + } + } + + // Add roles user should have but doesn't + for (const roleId of shouldHaveRoleIds) { + if (!existingRoleIds.has(roleId)) { + await assignUserRole(user.id, roleId, undefined); + } + } + + // Remove mapped roles user has but shouldn't + for (const roleId of mappedRoleIds) { + if (existingRoleIds.has(roleId) && !shouldHaveRoleIds.has(roleId)) { + await removeUserRole(user.id, roleId); } } } @@ -744,8 +784,9 @@ async function checkLdapGroupMembership( let groupFilter: string; if (config.groupFilter) { - // User provided custom filter - searchBase = config.groupBaseDn || groupDnOrName; + // User provided custom filter - use group DN directly when it's a full DN + // to avoid searching all groups under groupBaseDn + searchBase = isFullDn ? groupDnOrName : (config.groupBaseDn || groupDnOrName); groupFilter = config.groupFilter .replace('{{username}}', userDn) .replace('{{user_dn}}', userDn) @@ -765,7 +806,7 @@ async function checkLdapGroupMembership( } const { searchEntries } = await client.search(searchBase, { - scope: isFullDn && !config.groupFilter ? 'base' : 'sub', + scope: isFullDn ? 'base' : 'sub', filter: groupFilter, sizeLimit: 1 }); @@ -825,9 +866,7 @@ function generateBackupCodes(): string[] { async function hashBackupCode(code: string): Promise { // Normalize: uppercase, remove spaces and dashes const normalized = code.toUpperCase().replace(/[\s-]/g, ''); - const hasher = new Bun.CryptoHasher('sha256'); - hasher.update(normalized); - return hasher.digest('hex'); + return createHash('sha256').update(normalized).digest('hex'); } /** @@ -1172,9 +1211,7 @@ async function getOidcDiscovery(issuerUrl: string): Promise { const env = await getEnvironment(id); if (!env) return false; + // Clean up in-memory metrics + const { clearEnvironmentMetrics } = await import('./metrics-store.js'); + clearEnvironmentMetrics(id); + // Clean up related records that don't have cascade delete defined try { await db.delete(hostMetrics).where(eq(hostMetrics.environmentId, id)); @@ -576,45 +579,28 @@ export async function saveHostMetric( memoryPercent: number, memoryUsed: number, memoryTotal: number, - environmentId?: number + environmentId?: number, + _skipEnvCheck = false ): Promise { - // Verify environment exists before inserting (avoids FK violations on deleted envs) - if (environmentId) { - const env = await getEnvironment(environmentId); - if (!env) return; - } - - await db.insert(hostMetrics).values({ - environmentId: environmentId || null, - cpuPercent, - memoryPercent, - memoryUsed, - memoryTotal - }); - - // Cleanup old metrics (keep last 24 hours) - const cutoff24h = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); - await db.delete(hostMetrics).where(sql`timestamp < ${cutoff24h}`); + // Delegated to in-memory ring buffer (no DB writes) + if (!environmentId) return; + const { pushMetric } = await import('./metrics-store.js'); + pushMetric(environmentId, cpuPercent, memoryPercent, memoryUsed, memoryTotal); } export async function getHostMetrics(limit = 60, environmentId?: number): Promise { if (environmentId) { - return db.select().from(hostMetrics) - .where(eq(hostMetrics.environmentId, environmentId)) - .orderBy(desc(hostMetrics.timestamp)) - .limit(limit); + const { getMetricsHistory } = await import('./metrics-store.js'); + // getMetricsHistory returns oldest-first, but callers expect newest-first + return getMetricsHistory(environmentId, limit).reverse(); } - return db.select().from(hostMetrics) - .orderBy(desc(hostMetrics.timestamp)) - .limit(limit); + const { getAllMetrics } = await import('./metrics-store.js'); + return getAllMetrics(limit); } export async function getLatestHostMetrics(environmentId: number): Promise { - const results = await db.select().from(hostMetrics) - .where(eq(hostMetrics.environmentId, environmentId)) - .orderBy(desc(hostMetrics.timestamp)) - .limit(1); - return results[0] ?? null; + const { getLatestMetric } = await import('./metrics-store.js'); + return getLatestMetric(environmentId); } // ============================================================================= @@ -3269,20 +3255,23 @@ export interface ContainerEventResult { offset: number; } -export async function logContainerEvent(data: ContainerEventCreateData): Promise { - // Timestamp is already an ISO-8601 string from event-subprocess - // Both SQLite and PostgreSQL schemas use mode: 'string' so we pass it directly - const result = await db.insert(containerEvents).values({ +export async function logContainerEvent( + data: ContainerEventCreateData +): Promise { + const attrs = data.actorAttributes ? JSON.stringify(data.actorAttributes) : null; + + const [inserted] = await db.insert(containerEvents).values({ environmentId: data.environmentId ?? null, containerId: data.containerId, containerName: data.containerName ?? null, image: data.image ?? null, action: data.action, - actorAttributes: data.actorAttributes ? JSON.stringify(data.actorAttributes) : null, + actorAttributes: attrs, timestamp: data.timestamp - }).returning(); + }).returning({ id: containerEvents.id }); - return getContainerEvent(result[0].id) as Promise; + const event = await getContainerEvent(inserted.id); + return event!; } export async function getContainerEvent(id: number): Promise { @@ -4502,11 +4491,16 @@ export async function setStackEnvVars( )); } - // Insert new vars + // Insert new vars (deduplicate by key - last entry wins) if (variables.length > 0) { + const seen = new Map(); + for (const v of variables) { + seen.set(v.key, v); + } + const deduped = Array.from(seen.values()); const now = new Date().toISOString(); await db.insert(stackEnvironmentVariables).values( - variables.map(v => ({ + deduped.map(v => ({ stackName, environmentId, key: v.key, diff --git a/src/lib/server/db/connection.ts b/src/lib/server/db/connection.ts deleted file mode 100644 index 27abb17..0000000 --- a/src/lib/server/db/connection.ts +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Database Connection Module - * - * Provides a unified database connection using Bun's SQL API. - * Supports both SQLite (default) and PostgreSQL (via DATABASE_URL). - */ - -import { SQL } from 'bun'; -import { existsSync, mkdirSync, readFileSync } from 'node:fs'; -import { join, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const __dirname = dirname(fileURLToPath(import.meta.url)); - -// Database configuration -const databaseUrl = process.env.DATABASE_URL; -const dataDir = process.env.DATA_DIR || './data'; - -// Detect database type -export const isPostgres = databaseUrl && (databaseUrl.startsWith('postgres://') || databaseUrl.startsWith('postgresql://')); -export const isSqlite = !isPostgres; - -/** - * Read a SQL file from the appropriate sql directory. - */ -function readSql(filename: string): string { - const sqlDir = isPostgres ? 'postgres' : 'sqlite'; - return readFileSync(join(__dirname, sqlDir, 'sql', filename), 'utf-8'); -} - -/** - * Validate PostgreSQL connection URL format. - */ -function validatePostgresUrl(url: string): void { - try { - const parsed = new URL(url); - - if (parsed.protocol !== 'postgres:' && parsed.protocol !== 'postgresql:') { - exitWithError(`Invalid protocol "${parsed.protocol}". Expected "postgres:" or "postgresql:"`, url); - } - - if (!parsed.hostname) { - exitWithError('Missing hostname in DATABASE_URL', url); - } - - if (!parsed.pathname || parsed.pathname === '/') { - exitWithError('Missing database name in DATABASE_URL', url); - } - } catch { - exitWithError('Invalid URL format', url); - } -} - -/** - * Print connection error and exit. - */ -function exitWithError(error: string, url?: string): never { - console.error('\n' + '='.repeat(70)); - console.error('DATABASE CONNECTION ERROR'); - console.error('='.repeat(70)); - console.error(`\nError: ${error}`); - - if (url) { - try { - const parsed = new URL(url); - if (parsed.password) parsed.password = '***'; - console.error(`\nProvided URL: ${parsed.toString()}`); - } catch { - console.error(`\nProvided URL: ${url.replace(/:[^:@]+@/, ':***@')}`); - } - } - - console.error('\n' + '-'.repeat(70)); - console.error('DATABASE_URL format:'); - console.error('-'.repeat(70)); - console.error('\n postgres://USER:PASSWORD@HOST:PORT/DATABASE'); - console.error('\nExamples:'); - console.error(' postgres://dockhand:secret@localhost:5432/dockhand'); - console.error(' postgres://admin:p4ssw0rd@192.168.1.100:5432/dockhand'); - console.error(' postgresql://user:pass@db.example.com/mydb?sslmode=require'); - console.error('\n' + '-'.repeat(70)); - console.error('To use SQLite instead, remove the DATABASE_URL environment variable.'); - console.error('='.repeat(70) + '\n'); - - process.exit(1); -} - -/** - * Create the database connection. - */ -function createConnection(): SQL { - if (isPostgres) { - // Validate PostgreSQL URL - validatePostgresUrl(databaseUrl!); - - console.log('Connecting to PostgreSQL database...'); - try { - const sql = new SQL(databaseUrl!); - return sql; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - exitWithError(`Failed to connect to PostgreSQL: ${message}`, databaseUrl); - } - } else { - // SQLite: Ensure db directory exists - const dbDir = join(dataDir, 'db'); - if (!existsSync(dbDir)) { - mkdirSync(dbDir, { recursive: true }); - } - - const dbPath = join(dbDir, 'dockhand.db'); - console.log(`Using SQLite database at: ${dbPath}`); - - const sql = new SQL(`sqlite://${dbPath}`); - - // Enable WAL mode for better performance - sql.run('PRAGMA journal_mode = WAL'); - - return sql; - } -} - -/** - * Initialize the database schema. - */ -async function initializeSchema(sql: SQL): Promise { - try { - // Create schema (tables) - await sql.run(readSql('schema.sql')); - - // Create indexes - await sql.run(readSql('indexes.sql')); - - // Insert seed data - await sql.run(readSql('seed.sql')); - - // Update system roles - await sql.run(readSql('system-roles.sql')); - - // Run maintenance - await sql.run(readSql('maintenance.sql')); - - console.log(`Database initialized successfully (${isPostgres ? 'PostgreSQL' : 'SQLite'})`); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error('Failed to initialize database schema:', message); - throw error; - } -} - -// Create and export the database connection -export const sql = createConnection(); - -// Initialize schema (runs async but we handle it) -initializeSchema(sql).catch((error) => { - const errorMsg = error instanceof Error ? error.message : String(error); - console.error('[DB] Database initialization failed:', errorMsg); - process.exit(1); -}); - -/** - * Helper to convert SQLite integer booleans to JS booleans. - * PostgreSQL returns actual booleans, SQLite returns 0/1. - */ -export function toBool(value: any): boolean { - if (typeof value === 'boolean') return value; - return Boolean(value); -} - -/** - * Helper to convert JS boolean to database value. - * PostgreSQL uses boolean, SQLite uses 0/1. - */ -export function fromBool(value: boolean): boolean | number { - return isPostgres ? value : (value ? 1 : 0); -} diff --git a/src/lib/server/db/drizzle.ts b/src/lib/server/db/drizzle.ts index 1c48e7c..9f55171 100644 --- a/src/lib/server/db/drizzle.ts +++ b/src/lib/server/db/drizzle.ts @@ -208,11 +208,11 @@ function readMigrationJournal(migrationsFolder: string): MigrationJournal | null async function getAppliedMigrations(client: any, postgres: boolean): Promise { try { if (postgres) { - // PostgreSQL using Bun.sql - note the 'drizzle' schema + // PostgreSQL using postgres-js - note the 'drizzle' schema const result = await client`SELECT hash, created_at FROM drizzle.__drizzle_migrations ORDER BY id`; return result.map((r: any) => ({ hash: r.hash, createdAt: r.created_at })); } else { - // SQLite using bun:sqlite + // SQLite using better-sqlite3 const stmt = client.prepare('SELECT hash, created_at FROM __drizzle_migrations ORDER BY id'); return stmt.all().map((r: any) => ({ hash: r.hash, createdAt: r.created_at })); } @@ -484,10 +484,10 @@ async function runMigrations( // Run migrations try { if (postgres) { - const { migrate } = await import('drizzle-orm/bun-sql/migrator'); + const { migrate } = await import('drizzle-orm/postgres-js/migrator'); await migrate(database, { migrationsFolder }); } else { - const { migrate } = await import('drizzle-orm/bun-sqlite/migrator'); + const { migrate } = await import('drizzle-orm/better-sqlite3/migrator'); await migrate(database, { migrationsFolder }); } @@ -605,7 +605,7 @@ async function initializeDatabase() { logHeader('DATABASE INITIALIZATION'); if (isPostgres) { - // PostgreSQL via postgres-js (more stable than bun:sql for concurrent queries) + // PostgreSQL via postgres-js validatePostgresUrl(config.databaseUrl!); logInfo(`Database: PostgreSQL`); @@ -634,7 +634,7 @@ async function initializeDatabase() { handleMigrationFailure(result.error, true); } } else { - // SQLite via bun:sqlite + // SQLite via better-sqlite3 const dbDir = join(config.dataDir, 'db'); if (!existsSync(dbDir)) { mkdirSync(dbDir, { recursive: true }); @@ -645,8 +645,8 @@ async function initializeDatabase() { logInfo(`Database: SQLite`); logInfo(`Path: ${dbPath}`); - const { drizzle } = await import('drizzle-orm/bun-sqlite'); - const { Database } = await import('bun:sqlite'); + const { drizzle } = await import('drizzle-orm/better-sqlite3'); + const Database = (await import('better-sqlite3')).default; // Import SQLite schema schema = await import('./schema/index.js'); @@ -655,11 +655,11 @@ async function initializeDatabase() { rawClient = new Database(dbPath); // Enable WAL mode for better performance and concurrency - rawClient.exec('PRAGMA journal_mode = WAL'); + rawClient.pragma('journal_mode = WAL'); // Synchronous NORMAL is a good balance between safety and speed - rawClient.exec('PRAGMA synchronous = NORMAL'); + rawClient.pragma('synchronous = NORMAL'); // Increase busy timeout to handle concurrent access (5 seconds) - rawClient.exec('PRAGMA busy_timeout = 5000'); + rawClient.pragma('busy_timeout = 5000'); db = drizzle({ client: rawClient, schema }); logSuccess('SQLite database opened'); diff --git a/src/lib/server/docker.ts b/src/lib/server/docker.ts index 261ba91..c3fceab 100644 --- a/src/lib/server/docker.ts +++ b/src/lib/server/docker.ts @@ -2,12 +2,15 @@ * Docker Operations Module * * Uses direct Docker API calls over Unix socket or HTTP/HTTPS. - * No external dependencies like dockerode - uses native Bun fetch. + * No external dependencies like dockerode - uses Node.js fetch. */ import { homedir } from 'node:os'; import { existsSync, mkdirSync, rmSync, readdirSync } from 'node:fs'; import { join, resolve } from 'node:path'; +import * as http from 'node:http'; +import * as https from 'node:https'; +import { createHash } from 'node:crypto'; import type { Environment } from './db'; import { getStackEnvVarsAsRecord } from './db'; import { isSystemContainer } from './scheduler/tasks/update-utils'; @@ -28,7 +31,7 @@ export class EnvironmentNotFoundError extends Error { /** * Custom error for Docker connection failures with user-friendly messages. - * Wraps raw Bun fetch errors to hide technical details from users. + * Wraps raw fetch errors to hide technical details from users. */ export class DockerConnectionError extends Error { public readonly originalError: unknown; @@ -279,6 +282,12 @@ const envCache = new Map(); // Cache TTL: 30 minutes (in milliseconds) const CACHE_TTL = 30 * 60 * 1000; +// All known Docker Hub hostname variations for credential matching +const DOCKER_HUB_HOSTS = new Set([ + 'docker.io', 'hub.docker.com', 'registry.hub.docker.com', + 'index.docker.io', 'registry-1.docker.io', 'registry.docker.io', 'docker.com' +]); + // Cleanup stale cache entries periodically function cleanupEnvCache() { const now = Date.now(); @@ -300,6 +309,199 @@ if (!globalThis.__dockerEnvCacheCleanupInterval) { globalThis.__dockerEnvCacheCleanupInterval = setInterval(cleanupEnvCache, 10 * 60 * 1000); } +// ============================================================================= +// Per-environment HTTPS Agent pool +// ============================================================================= +// Used as a fallback when the Go TLS proxy is not available. +// Node's https.Agent with keepAlive reuses connections properly. +// ============================================================================= + +interface CachedAgent { + agent: https.Agent; + lastUsed: number; +} + +const agentCache = new Map(); + +function getHttpsAgent(config: DockerClientConfig): https.Agent { + // Hash actual cert content so rotated certs get a new agent + const h = createHash('sha256'); + h.update(`${config.host}:${config.port}:`); + if (config.ca) h.update(`ca:${config.ca}`); + if (config.cert) h.update(`cert:${config.cert}`); + if (config.key) h.update(`key:${config.key}`); + if (config.skipVerify) h.update('skip'); + const key = h.digest('hex'); + + const cached = agentCache.get(key); + if (cached) { + cached.lastUsed = Date.now(); + return cached.agent; + } + + const agentOptions: https.AgentOptions = { + keepAlive: true, + timeout: 30000, + }; + + if (config.ca) agentOptions.ca = config.ca; + if (config.cert) agentOptions.cert = config.cert; + if (config.key) agentOptions.key = config.key; + if (config.skipVerify) agentOptions.rejectUnauthorized = false; + + const agent = new https.Agent(agentOptions); + agentCache.set(key, { agent, lastUsed: Date.now() }); + return agent; +} + +function cleanupAgentCache() { + const now = Date.now(); + for (const [key, cached] of agentCache.entries()) { + if (now - cached.lastUsed > CACHE_TTL) { + cached.agent.destroy(); + agentCache.delete(key); + } + } +} + +declare global { + var __dockerAgentCacheCleanupInterval: ReturnType | undefined; +} + +if (!globalThis.__dockerAgentCacheCleanupInterval) { + globalThis.__dockerAgentCacheCleanupInterval = setInterval(cleanupAgentCache, 10 * 60 * 1000); +} + +/** + * Make an HTTPS request using Node.js https module with persistent Agent. + * Supports both buffered and streaming response modes. + */ +export function httpsAgentRequest( + config: DockerClientConfig, + path: string, + options: RequestInit = {}, + streaming: boolean = false, + extraHeaders?: Record +): Promise { + return new Promise((resolve, reject) => { + const method = (options.method || 'GET').toUpperCase(); + const agent = getHttpsAgent(config); + + const reqHeaders: Record = { ...(extraHeaders || {}) }; + if (options.headers) { + if (options.headers instanceof Headers) { + options.headers.forEach((value, key) => { reqHeaders[key] = value; }); + } else if (typeof options.headers === 'object') { + Object.assign(reqHeaders, options.headers); + } + } + + const reqOptions: https.RequestOptions = { + hostname: config.host, + port: config.port, + path, + method, + agent, + headers: reqHeaders, + }; + + if (!streaming) { + const isComposeOperation = path === '/_hawser/compose'; + const composeTimeoutMs = parseInt(process.env.COMPOSE_TIMEOUT || '900') * 1000; + reqOptions.timeout = isComposeOperation ? composeTimeoutMs : 30000; + } + + // Honor AbortSignal from caller (e.g., AbortSignal.timeout(5000) for ping) + const signal = options.signal as AbortSignal | undefined; + if (signal?.aborted) { + reject(new Error('Request aborted')); + return; + } + + const req = https.request(reqOptions, (res) => { + const headers = new Headers(); + for (const [key, value] of Object.entries(res.headers)) { + if (value) { + if (Array.isArray(value)) { + value.forEach(v => headers.append(key, v)); + } else { + headers.set(key, value); + } + } + } + + const status = res.statusCode || 200; + const statusText = res.statusMessage || ''; + + // Status codes that must not have a body + if ([101, 204, 205, 304].includes(status)) { + resolve(new Response(null, { status, statusText, headers })); + res.resume(); // drain + return; + } + + if (streaming) { + const readable = new ReadableStream({ + start(controller) { + res.on('data', (chunk: Buffer) => controller.enqueue(new Uint8Array(chunk))); + res.on('end', () => controller.close()); + res.on('error', (err) => controller.error(err)); + }, + cancel() { res.destroy(); } + }); + resolve(new Response(readable, { status, statusText, headers })); + } else { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => { + resolve(new Response(Buffer.concat(chunks), { status, statusText, headers })); + }); + res.on('error', reject); + } + }); + + req.on('error', reject); + req.on('timeout', () => { req.destroy(new Error('Request timeout')); }); + + if (signal) { + signal.addEventListener('abort', () => { + req.destroy(new Error('Request aborted')); + }, { once: true }); + } + + const body = options.body; + if (body) { + if (typeof body === 'string') { + req.end(body); + } else if (Buffer.isBuffer(body) || body instanceof Uint8Array) { + req.end(body); + } else if (body instanceof ArrayBuffer) { + req.end(Buffer.from(body)); + } else if (body instanceof Blob) { + body.arrayBuffer().then(ab => req.end(Buffer.from(ab)), reject); + } else if (typeof (body as ReadableStream).getReader === 'function') { + const reader = (body as ReadableStream).getReader(); + (async () => { + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + req.write(value); + } + req.end(); + } catch (err) { + req.destroy(err as Error); + } + })(); + } else { + req.end(); + } + } else { + req.end(); + } + }); +} + // Import db functions for environment lookup import { getEnvironment } from './db'; @@ -309,7 +511,7 @@ import { sendEdgeRequest, sendEdgeStreamRequest, isEdgeConnected, type EdgeRespo /** * Docker API client configuration */ -interface DockerClientConfig { +export interface DockerClientConfig { type: 'socket' | 'http' | 'https'; socketPath?: string; host?: string; @@ -341,6 +543,7 @@ function buildConfigFromEnv(env: Environment): DockerClientConfig { // Direct or Hawser connection types - use HTTP/HTTPS const protocol = (env.protocol as 'http' | 'https') || 'http'; + return { type: protocol, host: env.host || 'localhost', @@ -381,7 +584,7 @@ async function getDockerConfig(envId?: number | null): Promise { } } +/** + * Drain a Docker API response, throwing if the status is not OK. + * Extracts the error message from the JSON body if available. + * Accepts optional extra status codes to treat as success (e.g. 304 Not Modified). + */ +async function throwDockerError(response: Response): Promise { + const body = await response.text().catch(() => ''); + let msg = `Docker API error: HTTP ${response.status}`; + if (body) { + try { msg = JSON.parse(body).message ?? body; } catch { msg = body; } + } + throw new Error(msg); +} + +async function assertDockerResponse(response: Response, ...acceptStatuses: number[]): Promise { + if (response.ok || acceptStatuses.includes(response.status)) { + await drainResponse(response); + return; + } + await throwDockerError(response); +} + /** * Make a request to the Docker API * Exported for use by stacks.ts module */ +const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]); + +/** + * Build a Web API Response, handling null-body status codes. + */ +function buildResponse(body: Buffer | ReadableStream, status: number, statusText: string, headers: Headers): Response { + if (NULL_BODY_STATUSES.has(status)) { + return new Response(null, { status, statusText, headers }); + } + return new Response(body, { status, statusText, headers }); +} + +/** + * Make an HTTP request over a Unix socket and return a Web API Response. + * Make an HTTP request over a Unix socket, returning a buffered Web API Response. + */ +export function unixSocketRequest( + socketPath: string, + path: string, + options: RequestInit = {} +): Promise { + return new Promise((resolve, reject) => { + const method = (options.method || 'GET').toUpperCase(); + + const reqOptions: http.RequestOptions = { + socketPath, + path, + method, + headers: {}, + }; + + if (options.headers) { + if (options.headers instanceof Headers) { + options.headers.forEach((value, key) => { + (reqOptions.headers as Record)[key] = value; + }); + } else if (typeof options.headers === 'object') { + Object.assign(reqOptions.headers!, options.headers); + } + } + + const req = http.request(reqOptions, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => { + const body = Buffer.concat(chunks); + const headers = new Headers(); + for (const [key, value] of Object.entries(res.headers)) { + if (value) { + if (Array.isArray(value)) { + value.forEach(v => headers.append(key, v)); + } else { + headers.set(key, value); + } + } + } + resolve(buildResponse(body, res.statusCode || 200, res.statusMessage || '', headers)); + }); + res.on('error', reject); + }); + + req.on('error', reject); + + if (options.body) { + if (typeof options.body === 'string') { + req.write(options.body); + } else if (options.body instanceof Uint8Array || Buffer.isBuffer(options.body)) { + req.write(options.body); + } + } + + req.end(); + }); +} + +/** + * Make an HTTP request over a Unix socket and return a streaming Web API Response. + * Used for long-lived connections like Docker events, logs, stats streaming. + */ +export function unixSocketStreamRequest( + socketPath: string, + path: string, + options: RequestInit = {} +): Promise { + return new Promise((resolve, reject) => { + const method = (options.method || 'GET').toUpperCase(); + + const reqOptions: http.RequestOptions = { + socketPath, + path, + method, + headers: {}, + }; + + if (options.headers) { + if (options.headers instanceof Headers) { + options.headers.forEach((value, key) => { + (reqOptions.headers as Record)[key] = value; + }); + } else if (typeof options.headers === 'object') { + Object.assign(reqOptions.headers!, options.headers); + } + } + + const req = http.request(reqOptions, (res) => { + const headers = new Headers(); + for (const [key, value] of Object.entries(res.headers)) { + if (value) { + if (Array.isArray(value)) { + value.forEach(v => headers.append(key, v)); + } else { + headers.set(key, value); + } + } + } + + const readable = new ReadableStream({ + start(controller) { + res.on('data', (chunk: Buffer) => { + controller.enqueue(new Uint8Array(chunk)); + }); + res.on('end', () => { + controller.close(); + }); + res.on('error', (err) => { + controller.error(err); + }); + }, + cancel() { + res.destroy(); + } + }); + + resolve(new Response(readable, { + status: res.statusCode || 200, + statusText: res.statusMessage || '', + headers, + })); + }); + + req.on('error', reject); + + if (options.body) { + if (typeof options.body === 'string') { + req.write(options.body); + } else if (options.body instanceof Uint8Array || Buffer.isBuffer(options.body)) { + req.write(options.body); + } + } + + req.end(); + }); +} + export async function dockerFetch( path: string, options: DockerFetchOptions = {}, @@ -438,10 +817,6 @@ export async function dockerFetch( const { streaming, ...fetchOptions } = options; const method = (options.method || 'GET').toUpperCase(); - // For streaming connections, disable Bun's idle timeout - // This prevents long-lived streams (like Docker events) from being terminated - const bunOptions = streaming ? { timeout: false } : {}; - // Hawser Edge mode - route through WebSocket connection if (config.connectionType === 'hawser-edge' && config.environmentId) { // Check if agent is connected @@ -468,6 +843,7 @@ export async function dockerFetch( // Parse body if present let body: unknown; + let isBinary = false; if (fetchOptions.body) { if (typeof fetchOptions.body === 'string') { try { @@ -475,6 +851,13 @@ export async function dockerFetch( } catch { body = fetchOptions.body; } + } else if (fetchOptions.body instanceof ArrayBuffer || fetchOptions.body instanceof Uint8Array) { + // Binary body (tar uploads etc.) — base64 encode for JSON transport + const bytes = fetchOptions.body instanceof ArrayBuffer + ? new Uint8Array(fetchOptions.body) + : fetchOptions.body; + body = Buffer.from(bytes).toString('base64'); + isBinary = true; } else { body = fetchOptions.body; } @@ -489,7 +872,8 @@ export async function dockerFetch( body, headers, streaming || false, - (streaming || path === '/_hawser/compose') ? 300000 : 30000 // 5 min for streaming/compose, 30s for normal + (streaming || path === '/_hawser/compose') ? 300000 : 30000, // 5 min for streaming/compose, 30s for normal + isBinary ); const elapsed = Date.now() - startTime; // Only warn for slow requests, but skip /stats which is expected to be slow (5-10s) @@ -507,15 +891,10 @@ export async function dockerFetch( } if (config.type === 'socket') { - // Use Bun's native Unix socket support - const url = `http://localhost${path}`; + // Unix socket via http.request({ socketPath }) try { - const response = await fetch(url, { - ...fetchOptions, - // @ts-ignore - Bun supports unix socket and timeout options - unix: config.socketPath, - ...bunOptions - }); + const requestFn = streaming ? unixSocketStreamRequest : unixSocketRequest; + const response = await requestFn(config.socketPath!, path, fetchOptions); const elapsed = Date.now() - startTime; // Only warn for slow requests, but skip /stats which is expected to be slow (5-10s) if (elapsed > 5000 && !path.includes('/stats')) { @@ -537,95 +916,49 @@ export async function dockerFetch( const finalOptions: RequestInit = { ...fetchOptions }; // For Hawser Standard mode with token authentication + const extraHeaders: Record = {}; if (config.connectionType === 'hawser-standard' && config.hawserToken) { + extraHeaders['X-Hawser-Token'] = config.hawserToken; finalOptions.headers = { ...finalOptions.headers, 'X-Hawser-Token': config.hawserToken }; } - // For HTTPS with TLS certificates, we need to configure TLS - // Pass certificate strings directly to Bun's fetch - no temp files needed + // For HTTPS: use node:https with persistent Agent (fallback when Go proxy is down). + // For plain HTTP: use standard fetch(). if (config.type === 'https') { - const tlsOptions: Record = {}; - - // Detect if mutual TLS (client certificate authentication) is in use - const isMtls = !!(config.cert && config.key); - - if (isMtls) { - // mTLS: Disable session caching to prevent Bun from reusing a TLS session - // with wrong client certificates (pool key doesn't include certs) - tlsOptions.sessionTimeout = 0; - } else { - // Non-mTLS HTTPS (CA-only or skip-verify): Allow short-lived session reuse. - // Without this, every fetch allocates a new native TLS context in BoringSSL. - // Native memory (mmap) is never returned to the OS, causing RSS to grow - // continuously in long-running subprocesses (metrics, events). - // 30s allows sessions to be reused within one metrics cycle, then expire. - tlsOptions.sessionTimeout = 30; - } - - // Set explicit servername for SNI - isolates TLS contexts per host - tlsOptions.servername = config.host; - - // Load CA certificate (just this environment's CA, not composite) - if (config.ca) { - tlsOptions.ca = [config.ca]; - } - - // Client cert and key for mTLS authentication - if (config.cert) { - tlsOptions.cert = [config.cert]; - } - if (config.key) { - tlsOptions.key = config.key; - } - - // Skip verification (self-signed without CA) - if (config.skipVerify) { - tlsOptions.rejectUnauthorized = false; - } else { - tlsOptions.rejectUnauthorized = true; - } - - if (Object.keys(tlsOptions).length > 0) { - // @ts-ignore - Bun supports tls options with string certs - finalOptions.tls = tlsOptions; - if (isMtls) { - // mTLS: Force new connection for each request to prevent Bun from - // reusing a TLS session with wrong client certificates - // @ts-ignore - Bun supports keepalive option - finalOptions.keepalive = false; + try { + const response = await httpsAgentRequest(config, path, finalOptions, streaming || false, extraHeaders); + const elapsed = Date.now() - startTime; + if (elapsed > 5000 && !path.includes('/stats')) { + console.warn(`[Docker] ${config.connectionType || 'direct'} ${config.host}: ${method} ${path} took ${elapsed}ms`); } - // Non-mTLS: Use Bun's default keepalive (connection reuse) to avoid - // allocating a new native TLS context per request - } - - // Optional verbose TLS debugging - if (process.env.DEBUG_TLS) { - // @ts-ignore - Bun-specific verbose option - finalOptions.verbose = true; + return response; + } catch (error: any) { + const elapsed = Date.now() - startTime; + const msg = error?.message || String(error); + console.error(`[Docker] ${config.connectionType || 'direct'} ${config.host}: ${method} ${path} failed after ${elapsed}ms: ${msg}`); + throw DockerConnectionError.fromError(error); } } - // Add default timeout for non-streaming requests to prevent socket accumulation - // Compose operations need more time (up to 5 minutes) for multi-service stacks + // Plain HTTP — use standard fetch() if (!streaming && !finalOptions.signal) { const isComposeOperation = path === '/_hawser/compose'; - finalOptions.signal = AbortSignal.timeout(isComposeOperation ? 300000 : 30000); + const composeTimeoutMs = parseInt(process.env.COMPOSE_TIMEOUT || '900') * 1000; + finalOptions.signal = AbortSignal.timeout(isComposeOperation ? composeTimeoutMs : 30000); } try { - const response = await fetch(url, { ...finalOptions, ...bunOptions }); + const response = await fetch(url, finalOptions); const elapsed = Date.now() - startTime; - // Only warn for slow requests, but skip /stats which is expected to be slow (5-10s) if (elapsed > 5000 && !path.includes('/stats')) { console.warn(`[Docker] ${config.connectionType || 'direct'} ${config.host}: ${method} ${path} took ${elapsed}ms`); } return response; } catch (error: any) { const elapsed = Date.now() - startTime; - // Log error message only, not full stack trace const msg = error?.message || String(error); console.error(`[Docker] ${config.connectionType || 'direct'} ${config.host}: ${method} ${path} failed after ${elapsed}ms: ${msg}`); throw DockerConnectionError.fromError(error); @@ -674,6 +1007,11 @@ export function clearDockerClientCache(envId?: number) { } else { envCache.clear(); } + // Destroy HTTPS agents (TLS config may have changed) + for (const [key, cached] of agentCache.entries()) { + cached.agent.destroy(); + agentCache.delete(key); + } } export interface ContainerInfo { @@ -782,51 +1120,37 @@ export async function getContainerStats(id: string, envId?: number | null) { export async function startContainer(id: string, envId?: number | null) { const response = await dockerFetch(`/containers/${id}/start`, { method: 'POST' }, envId); - await drainResponse(response); + await assertDockerResponse(response, 304); // 304 = already started } export async function stopContainer(id: string, envId?: number | null) { const response = await dockerFetch(`/containers/${id}/stop`, { method: 'POST' }, envId); - await drainResponse(response); + await assertDockerResponse(response, 304); // 304 = already stopped } export async function restartContainer(id: string, envId?: number | null) { const response = await dockerFetch(`/containers/${id}/restart`, { method: 'POST' }, envId); - await drainResponse(response); + await assertDockerResponse(response); } export async function pauseContainer(id: string, envId?: number | null) { const response = await dockerFetch(`/containers/${id}/pause`, { method: 'POST' }, envId); - await drainResponse(response); + await assertDockerResponse(response); } export async function unpauseContainer(id: string, envId?: number | null) { const response = await dockerFetch(`/containers/${id}/unpause`, { method: 'POST' }, envId); - await drainResponse(response); + await assertDockerResponse(response); } export async function removeContainer(id: string, force = false, envId?: number | null) { const response = await dockerFetch(`/containers/${id}?force=${force}`, { method: 'DELETE' }, envId); - if (!response.ok && response.status !== 404) { - const errorBody = await response.text(); - let errorMessage = `Failed to remove container ${id}`; - try { - const parsed = JSON.parse(errorBody); - if (parsed.message) { - errorMessage = parsed.message; - } - } catch { - if (errorBody) { - errorMessage = errorBody; - } - } - throw new Error(errorMessage); - } + await assertDockerResponse(response, 404); // 404 = already gone } export async function renameContainer(id: string, newName: string, envId?: number | null) { const response = await dockerFetch(`/containers/${id}/rename?name=${encodeURIComponent(newName)}`, { method: 'POST' }, envId); - await drainResponse(response); + await assertDockerResponse(response); } export async function getContainerLogs(id: string, tail = 100, envId?: number | null): Promise { @@ -840,6 +1164,8 @@ export async function getContainerLogs(id: string, tail = 100, envId?: number | envId ); + if (!response.ok) await throwDockerError(response); + const buffer = Buffer.from(await response.arrayBuffer()); // If TTY is enabled, logs are raw text (no demux needed) @@ -1427,7 +1753,7 @@ export async function recreateContainerFromInspect( `/containers/${oldContainerId}/rename?name=${encodeURIComponent(name + '-old')}`, { method: 'POST' }, envId - ).then(r => { if (!r.ok) throw new Error('Failed to rename old container'); }); + ).then(async r => { if (!r.ok) throw new Error('Failed to rename old container'); await drainResponse(r); }); // 3. Disconnect all networks from old container (frees static IPs) // Skip for shared network modes (container:X, host, none) — Docker manages these @@ -1463,7 +1789,7 @@ export async function recreateContainerFromInspect( `/containers/${oldContainerId}/rename?name=${encodeURIComponent(name)}`, { method: 'POST' }, envId - ).catch(() => {}); + ).then(r => drainResponse(r)).catch(() => {}); // Reconnect networks using full EndpointSettings from inspect if (!isSharedNetwork) { @@ -1533,6 +1859,34 @@ export async function recreateContainerFromInspect( } } + // Deduplicate: remove Config.Volumes entries that conflict with HostConfig.Tmpfs or Binds. + // Read-only containers get tmpfs at paths like /tmp that may also be declared as image volumes. + // Docker rejects duplicate mount points, so the tmpfs/bind mount wins over the volume declaration. + if (createConfig.Volumes && hostConfig) { + const mountedPaths = new Set(); + if (hostConfig.Tmpfs) { + for (const p of Object.keys(hostConfig.Tmpfs)) { + mountedPaths.add(p); + } + } + if (hostConfig.Binds) { + for (const b of hostConfig.Binds) { + const parts = b.split(':'); + if (parts.length >= 2) mountedPaths.add(parts[1].split(':')[0]); + } + } + if (mountedPaths.size > 0) { + for (const volPath of Object.keys(createConfig.Volumes)) { + if (mountedPaths.has(volPath)) { + delete createConfig.Volumes[volPath]; + } + } + if (Object.keys(createConfig.Volumes).length === 0) { + delete createConfig.Volumes; + } + } + } + // Preserve anonymous volumes from Mounts not in HostConfig.Binds const existingBinds = new Set((hostConfig.Binds || []).map((b: string) => { const parts = b.split(':'); @@ -1557,10 +1911,8 @@ export async function recreateContainerFromInspect( // Docker can only connect to one network at creation. Pass the first network // from the old container's settings to avoid getting a random bridge IP. // Skip for shared network modes — EndpointsConfig conflicts with container:/host/none modes. - // Clear MacAddress for Docker API < 1.44 compatibility. if (!isSharedNetwork && initialNetworkName && initialNetworkConfig) { const endpointConfig = { ...initialNetworkConfig }; - delete endpointConfig.MacAddress; createConfig.NetworkingConfig = { EndpointsConfig: { [initialNetworkName]: endpointConfig @@ -1995,6 +2347,39 @@ export async function listImages(envId?: number | null): Promise { })); } +/** + * Build X-Registry-Auth header for authenticated Docker image pulls. + * Looks up stored registry credentials and returns a headers object + * with the base64-encoded auth config, or an empty object if no credentials found. + */ +export async function buildRegistryAuthHeader(imageName: string): Promise> { + const headers: Record = {}; + try { + const { registry } = parseImageReference(imageName); + const creds = await findRegistryCredentials(registry); + if (creds) { + // Docker Engine requires 'https://index.docker.io/v1/' as serveraddress + // for Docker Hub auth — just the hostname is treated as unauthenticated + const serveraddress = DOCKER_HUB_HOSTS.has(registry) + ? 'https://index.docker.io/v1/' + : registry; + console.log(`[Pull] Using credentials for ${serveraddress} (user: ${creds.username})`); + const authConfig = { + username: creds.username, + password: creds.password, + serveraddress + }; + headers['X-Registry-Auth'] = Buffer.from(JSON.stringify(authConfig)).toString('base64'); + } else { + console.log(`[Pull] No credentials found for ${registry}`); + } + } catch (e) { + const errorMsg = e instanceof Error ? e.message : String(e); + console.error(`[Pull] Failed to lookup credentials:`, errorMsg); + } + return headers; +} + export async function pullImage(imageName: string, onProgress?: (data: any) => void, envId?: number | null) { // Parse image name and tag to avoid pulling all tags // Docker API: if tag is empty, it pulls ALL tags for the image @@ -2025,26 +2410,7 @@ export async function pullImage(imageName: string, onProgress?: (data: any) => v : `/images/create?fromImage=${encodeURIComponent(fromImage)}`; // Look up registry credentials for authenticated pulls - const headers: Record = {}; - try { - const { registry } = parseImageReference(imageName); - const creds = await findRegistryCredentials(registry); - if (creds) { - console.log(`[Pull] Using credentials for ${registry} (user: ${creds.username})`); - // Docker API expects X-Registry-Auth header with base64-encoded JSON - const authConfig = { - username: creds.username, - password: creds.password, - serveraddress: registry - }; - headers['X-Registry-Auth'] = Buffer.from(JSON.stringify(authConfig)).toString('base64'); - } else { - console.log(`[Pull] No credentials found for ${registry}`); - } - } catch (e) { - const errorMsg = e instanceof Error ? e.message : String(e); - console.error(`[Pull] Failed to lookup credentials:`, errorMsg); - } + const headers = await buildRegistryAuthHeader(imageName); // Use streaming: true for longer timeout on edge environments const response = await dockerFetch(url, { method: 'POST', streaming: true, headers }, envId); @@ -2090,6 +2456,7 @@ export async function removeImage(id: string, force = false, envId?: number | nu error.json = data; throw error; } + await drainResponse(response); } export async function getImageHistory(id: string, envId?: number | null) { @@ -2210,14 +2577,12 @@ async function findRegistryCredentials(registryHost: string): Promise<{ username } } - // Also check for Docker Hub variations - if (requested.host === 'index.docker.io' || requested.host === 'registry-1.docker.io') { + // Bidirectional Docker Hub alias matching: + // If the requested host is any Docker Hub variant, match against any stored Docker Hub variant + if (DOCKER_HUB_HOSTS.has(requested.host)) { for (const reg of registries) { const stored = parseRegistryUrl(reg.url); - // Match all Docker Hub URL variations - if (stored.host === 'docker.io' || stored.host === 'hub.docker.com' || - stored.host === 'registry.hub.docker.com' || stored.host === 'index.docker.io' || - stored.host === 'registry-1.docker.io') { + if (DOCKER_HUB_HOSTS.has(stored.host)) { if (reg.username && reg.password) { return { username: reg.username, password: reg.password }; } @@ -2255,11 +2620,13 @@ async function getRegistryBearerToken(registry: string, repo: string): Promise { + try { + const response = await dockerFetch('/_ping', { + signal: AbortSignal.timeout(5000) + }, envId); + await drainResponse(response); + return response.ok; + } catch (error: any) { + const msg = error?.message || String(error); + if (msg.includes('unreachable')) { + const config = await getDockerConfig(envId).catch(() => null); + console.warn(`[Docker] ${config?.connectionType || 'direct'} ${config?.host || envId}: /_ping failed - host unreachable`); + } + return false; + } +} + /** * Get Hawser agent info (for hawser-standard mode) * Returns agent info including uptime @@ -2770,6 +3169,7 @@ export async function getHawserInfo(envId: number): Promise<{ if (response.ok) { return await response.json(); } + await drainResponse(response); console.warn(`[Hawser] Info endpoint returned ${response.status} for env ${envId}`); } catch (error) { const msg = error instanceof Error ? error.message : String(error); @@ -2867,6 +3267,7 @@ export async function removeVolume(name: string, force = false, envId?: number | error.json = data; throw error; } + await drainResponse(response); } export async function inspectVolume(name: string, envId?: number | null) { @@ -2962,6 +3363,7 @@ export async function removeNetwork(id: string, envId?: number | null) { error.json = data; throw error; } + await drainResponse(response); } export async function inspectNetwork(id: string, envId?: number | null) { @@ -3073,6 +3475,7 @@ export async function connectContainerToNetwork( const data = await response.json().catch(() => ({})); throw new Error(data.message || 'Failed to connect container to network'); } + await drainResponse(response); } /** @@ -3104,6 +3507,7 @@ export async function connectContainerToNetworkRaw( const data = await response.json().catch(() => ({})); throw new Error(data.message || 'Failed to connect container to network'); } + await drainResponse(response); } export async function disconnectContainerFromNetwork( @@ -3125,6 +3529,7 @@ export async function disconnectContainerFromNetwork( const data = await response.json().catch(() => ({})); throw new Error(data.message || 'Failed to disconnect container from network'); } + await drainResponse(response); } // Container exec operations @@ -3181,6 +3586,63 @@ export async function getDockerConnectionInfo(envId?: number | null): Promise<{ }; } +// ============================================================================= +// Global handlers for server.js terminal WebSocket connections +// ============================================================================= +// server.js cannot import SvelteKit modules directly, so we expose these +// functions via globalThis (same pattern as Hawser handlers). +// ============================================================================= + +declare global { + var __terminalGetTarget: ((envId?: number) => Promise<{ + type: 'socket' | 'http' | 'https'; + connectionType?: string; + socketPath?: string; + host?: string; + port?: number; + hawserToken?: string; + environmentId?: number; + tls?: { ca?: string; cert?: string; key?: string; rejectUnauthorized: boolean }; + }>) | undefined; + var __terminalCreateExec: ((containerId: string, shell: string, user: string, envId?: number) => Promise) | undefined; + var __terminalResizeExec: ((execId: string, cols: number, rows: number, envId?: number) => Promise) | undefined; +} + +globalThis.__terminalGetTarget = async (envId?: number) => { + if (!envId) { + // No environment = local socket + return { type: 'socket', connectionType: 'socket', socketPath: '/var/run/docker.sock' }; + } + const config = await getDockerConfig(envId); + const result: Awaited>> = { + type: config.type, + connectionType: config.connectionType, + socketPath: config.socketPath, + host: config.host, + port: config.port, + hawserToken: config.hawserToken, + environmentId: config.environmentId, + }; + if (config.type === 'https') { + result.tls = { + ca: config.ca, + cert: config.cert, + key: config.key, + rejectUnauthorized: !config.skipVerify, + }; + } + return result; +}; + +globalThis.__terminalCreateExec = async (containerId, shell, user, envId) => { + const exec = await createExec({ containerId, cmd: [shell], user, envId }); + return exec.Id; +}; + +globalThis.__terminalResizeExec = async (execId, cols, rows, envId) => { + await resizeExec(execId, cols, rows, envId); +}; + // System disk usage export async function getDiskUsage(envId?: number | null) { return dockerJsonRequest('/system/df', {}, envId); @@ -3275,6 +3737,8 @@ export async function execInContainer( envId ); + if (!response.ok) await throwDockerError(response); + const buffer = Buffer.from(await response.arrayBuffer()); const output = demuxDockerStream(buffer) as string; @@ -3313,9 +3777,8 @@ export async function getDockerEvents( } try { - // Note: We use streaming: true to disable Bun's idle timeout for this long-lived connection. + // Use streaming: true for this long-lived connection. // The Docker events API keeps the connection open indefinitely, sending events as they occur. - // Without streaming: true, Bun would terminate the connection after ~5 seconds of inactivity. const response = await dockerFetch( `/events?${queryString}`, { streaming: true }, @@ -3390,11 +3853,12 @@ export async function runContainer(options: { try { // Start container console.log(`[runContainer] Starting container ${containerId}...`); - await drainResponse(await dockerFetch(`/containers/${containerId}/start`, { method: 'POST' }, options.envId)); + await assertDockerResponse(await dockerFetch(`/containers/${containerId}/start`, { method: 'POST' }, options.envId)); // Wait for container to finish console.log(`[runContainer] Waiting for container ${containerId} to finish...`); const waitResponse = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST', streaming: true }, options.envId); + if (!waitResponse.ok) await throwDockerError(waitResponse); const waitResult = await waitResponse.json().catch(() => ({})); console.log(`[runContainer] Container ${containerId} finished with exit code:`, waitResult?.StatusCode); @@ -3406,6 +3870,8 @@ export async function runContainer(options: { options.envId ); + if (!logsResponse.ok) await throwDockerError(logsResponse); + const buffer = Buffer.from(await logsResponse.arrayBuffer()); console.log(`[runContainer] Got logs buffer, size: ${buffer.length} bytes`); @@ -3437,6 +3903,7 @@ export async function runContainerWithStreaming(options: { onStdout?: (data: string) => void; onStderr?: (data: string) => void; timeout?: number; // Overall timeout in ms (0 or undefined = no timeout) + networkMode?: string; // Docker network mode (e.g., network name for TCP access) }): Promise { const baseName = options.name || `dockhand-stream-${Date.now()}`; const containerName = `${baseName}-${randomSuffix()}`; @@ -3449,7 +3916,11 @@ export async function runContainerWithStreaming(options: { Tty: false, HostConfig: { Binds: options.binds || [], - AutoRemove: false + AutoRemove: false, + LogConfig: { + Type: 'json-file', + Config: {} + } } }; @@ -3458,6 +3929,11 @@ export async function runContainerWithStreaming(options: { containerConfig.User = options.user; } + // Set network mode if specified (e.g., for scanner containers accessing Docker via TCP) + if (options.networkMode) { + containerConfig.HostConfig.NetworkMode = options.networkMode; + } + const createResult = await dockerJsonRequest<{ Id: string }>( `/containers/create?name=${encodeURIComponent(containerName)}`, { method: 'POST', body: JSON.stringify(containerConfig) }, @@ -3487,6 +3963,7 @@ export async function runContainerWithStreaming(options: { let exitCode: number | undefined; try { const waitResult = await dockerFetch(`/containers/${containerId}/wait`, { method: 'POST', streaming: true }, options.envId); + if (!waitResult.ok) await throwDockerError(waitResult); const waitData = await waitResult.json() as { StatusCode?: number }; exitCode = waitData.StatusCode; console.log(`[runContainerWithStreaming] Container exited with code: ${exitCode}`); @@ -3560,6 +4037,8 @@ async function streamLocalStderr( envId ); + if (!response.ok) await throwDockerError(response); + const reader = response.body?.getReader(); if (!reader) return; @@ -3680,10 +4159,12 @@ async function fetchContainerStdout( // Local/standard mode - read via streaming to handle large Docker log responses const response = await dockerFetch( `/containers/${containerId}/logs?stdout=true&stderr=false&follow=false`, - {}, + { streaming: true }, envId ); + if (!response.ok) await throwDockerError(response); + const reader = response.body?.getReader(); if (!reader) { const buffer = Buffer.from(await response.arrayBuffer()); @@ -3721,6 +4202,7 @@ export async function pushImage( `/images/${encodeURIComponent(imageTag)}/push`, { method: 'POST', + streaming: true, headers: { 'X-Registry-Auth': authHeader } @@ -3782,7 +4264,7 @@ export interface FileEntry { /** * Parse ls -la output into FileEntry array * Handles multiple formats: - * - GNU ls with --time-style=iso: drwxr-xr-x 2 root root 4096 2024-12-08 10:30 dirname + * - GNU ls with --time-style=long-iso: drwxr-xr-x 2 root root 4096 2024-12-08 10:30 dirname * - Standard GNU ls: drwxr-xr-x 2 root root 4096 Dec 8 10:30 dirname * - Busybox ls: drwxr-xr-x 2 root root 4096 Dec 8 10:30 dirname */ @@ -3820,7 +4302,7 @@ function parseLsOutput(output: string): FileEntry[] { let time: string; let nameAndLink: string; - // Try ISO format first (GNU ls with --time-style=iso) + // Try ISO format first (GNU ls with --time-style=long-iso) // Format: drwxr-xr-x 2 root root 4096 2024-12-08 10:30 dirname // With ACL: drwxr-xr-x+ 2 root root 4096 2024-12-08 10:30 dirname // With extended attrs: drwxr-xr-x@ 2 root root 4096 2024-12-08 10:30 dirname @@ -3965,7 +4447,7 @@ export async function listContainerDirectory( ['/usr/bin/ls', '-la', safePath], ] : [ - ['ls', '-la', '--time-style=iso', safePath], + ['ls', '-la', '--time-style=long-iso', safePath], ['ls', '-la', safePath], ['/bin/ls', '-la', safePath], ['/usr/bin/ls', '-la', safePath], @@ -4001,7 +4483,7 @@ export async function getContainerArchive( const response = await dockerFetch( `/containers/${containerId}/archive?path=${encodeURIComponent(safePath)}`, - {}, + { streaming: true }, envId ); @@ -4041,6 +4523,7 @@ export async function putContainerArchive( const error = await response.text(); throw new Error(`Failed to upload archive: ${error}`); } + await drainResponse(response); } /** @@ -4061,6 +4544,7 @@ export async function statContainerPath( ); if (!response.ok) { + await drainResponse(response); throw new Error(`Path not found: ${safePath}`); } @@ -4291,14 +4775,16 @@ async function ensureVolumeHelperImage(envId?: number | null): Promise { const response = await dockerFetch(`/images/${encodeURIComponent(VOLUME_HELPER_IMAGE)}/json`, {}, envId); if (response.ok) { + await drainResponse(response); return; // Image exists } // Image not found, pull it console.log(`Pulling ${VOLUME_HELPER_IMAGE} for volume browsing...`); + const authHeaders = await buildRegistryAuthHeader(VOLUME_HELPER_IMAGE); const pullResponse = await dockerFetch( `/images/create?fromImage=${encodeURIComponent(VOLUME_HELPER_IMAGE)}`, - { method: 'POST' }, + { method: 'POST', headers: authHeaders }, envId ); @@ -4533,10 +5019,11 @@ async function cleanupStaleVolumeHelpersForEnv(envId?: number | null): Promise): Promise { - console.log('Cleaning up stale volume helper containers...'); - - if (!environments || environments.length === 0) { - console.log('No environments to clean up'); - return; - } + if (!environments || environments.length === 0) return; let totalRemoved = 0; - // Clean up all configured environments for (const env of environments) { totalRemoved += await cleanupStaleVolumeHelpersForEnv(env.id); } if (totalRemoved > 0) { - console.log(`Removed ${totalRemoved} stale volume helper container(s)`); + console.log(`[Volume Helper] Removed ${totalRemoved} stale container(s)`); } } @@ -4614,7 +5095,7 @@ export async function getVolumeArchive( const response = await dockerFetch( `/containers/${containerId}/archive?path=${encodeURIComponent(fullPath)}`, - {}, + { streaming: true }, envId ); diff --git a/src/lib/server/event-collector.ts b/src/lib/server/event-collector.ts index 8cc496a..da910ed 100644 --- a/src/lib/server/event-collector.ts +++ b/src/lib/server/event-collector.ts @@ -2,17 +2,25 @@ * Container Event Emitter * * Shared EventEmitter for broadcasting container events to SSE clients. - * Events are emitted by the subprocess-manager when it receives them from the event-subprocess. + * Events are emitted by the collection worker when processing Docker events. + * + * IMPORTANT: Uses globalThis to ensure a single instance across all module imports. + * In Vite dev mode and SvelteKit production builds, server modules can be loaded + * multiple times (HMR, chunking), creating separate EventEmitter instances. + * Using globalThis guarantees emitters and listeners share the same object. */ import { EventEmitter } from 'node:events'; -// Event emitter for broadcasting new events to SSE clients -// Used by: -// - subprocess-manager.ts: emits events received from event-subprocess via IPC -// - api/activity/events/+server.ts: listens for events to broadcast via SSE -export const containerEventEmitter = new EventEmitter(); +const GLOBAL_KEY = '__dockhand_container_event_emitter__'; -// Allow up to 100 concurrent SSE listeners (default is 10) -// This prevents MaxListenersExceededWarning with many dashboard clients -containerEventEmitter.setMaxListeners(100); +// Ensure single instance via globalThis +if (!(globalThis as any)[GLOBAL_KEY]) { + const emitter = new EventEmitter(); + // Allow up to 100 concurrent SSE listeners (default is 10) + // This prevents MaxListenersExceededWarning with many dashboard clients + emitter.setMaxListeners(100); + (globalThis as any)[GLOBAL_KEY] = emitter; +} + +export const containerEventEmitter: EventEmitter = (globalThis as any)[GLOBAL_KEY]; diff --git a/src/lib/server/git.ts b/src/lib/server/git.ts index cc6d931..294fe56 100644 --- a/src/lib/server/git.ts +++ b/src/lib/server/git.ts @@ -1,5 +1,7 @@ -import { existsSync, mkdirSync, rmSync, chmodSync } from 'node:fs'; +import { existsSync, mkdirSync, rmSync, chmodSync, readFileSync, writeFileSync } from 'node:fs'; import { join, resolve, dirname, basename, relative } from 'node:path'; +import { spawn as nodeSpawn, spawnSync } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; import { getGitRepository, getGitCredential, @@ -14,6 +16,26 @@ import { } from './db'; import { deployStack, getStackDir } from './stacks'; +/** + * Collect stdout, stderr and exit code from a spawned process. + */ +function collectProcess(proc: ChildProcess): Promise<{ exitCode: number; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + proc.stdout?.on('data', (chunk: Buffer) => stdoutChunks.push(chunk)); + proc.stderr?.on('data', (chunk: Buffer) => stderrChunks.push(chunk)); + proc.on('error', reject); + proc.on('close', (code) => { + resolve({ + exitCode: code ?? 1, + stdout: Buffer.concat(stdoutChunks).toString(), + stderr: Buffer.concat(stderrChunks).toString() + }); + }); + }); +} + // Directory for storing cloned repositories const dataDir = process.env.DATA_DIR || './data'; const GIT_REPOS_DIR = resolve(process.env.GIT_REPOS_DIR || join(dataDir, 'git-repos')); @@ -79,7 +101,7 @@ async function ensurePasswdEntry(env: GitEnv): Promise { if (uid === undefined || uid === 0) return; // root or not available try { - const passwd = await Bun.file('/etc/passwd').text(); + const passwd = readFileSync('/etc/passwd', 'utf-8'); const uidStr = `:${uid}:`; if (passwd.split('\n').some(line => { const parts = line.split(':'); @@ -100,17 +122,17 @@ async function ensurePasswdEntry(env: GitEnv): Promise { // Create temp passwd/group with the missing entry try { const gid = process.getgid?.() ?? uid; - const passwd = await Bun.file('/etc/passwd').text(); - const group = await Bun.file('/etc/group').text(); + const passwd = readFileSync('/etc/passwd', 'utf-8'); + const group = readFileSync('/etc/group', 'utf-8'); const passwdEntry = `dockhand:x:${uid}:${gid}:Dockhand:/home/dockhand:/bin/sh`; - await Bun.write(TMP_PASSWD, passwd.trimEnd() + '\n' + passwdEntry + '\n'); + writeFileSync(TMP_PASSWD, passwd.trimEnd() + '\n' + passwdEntry + '\n'); const gidExists = group.split('\n').some(line => line.split(':')[2] === String(gid)); if (gidExists) { - await Bun.write(TMP_GROUP, group); + writeFileSync(TMP_GROUP, group); } else { - await Bun.write(TMP_GROUP, group.trimEnd() + '\n' + `dockhand:x:${gid}:\n`); + writeFileSync(TMP_GROUP, group.trimEnd() + '\n' + `dockhand:x:${gid}:\n`); } _nssWrapperNeeded = true; @@ -135,8 +157,10 @@ async function buildGitEnv(credential: GitCredential | null): Promise { await ensurePasswdEntry(env); if (credential?.authType === 'ssh' && credential.sshPrivateKey) { - // Create a temporary SSH key file (use absolute path so SSH can find it) - const sshKeyPath = resolve(join(GIT_REPOS_DIR, `.ssh-key-${credential.id}`)); + // Write SSH key to /tmp instead of data volume — some filesystems (TrueNAS ZFS, + // NFS, CIFS) silently ignore chmod, leaving the key group-readable (e.g. 0670). + // SSH refuses keys that are accessible by others. /tmp is always a proper filesystem. + const sshKeyPath = `/tmp/.ssh-key-${credential.id}`; // Ensure SSH key ends with a newline (newer SSH versions are strict about this) let keyContent = credential.sshPrivateKey; @@ -144,18 +168,19 @@ async function buildGitEnv(credential: GitCredential | null): Promise { keyContent += '\n'; } - await Bun.write(sshKeyPath, keyContent); + writeFileSync(sshKeyPath, keyContent); // Ensure SSH key has correct permissions (0600 = owner read/write only) - // Bun.write's mode option doesn't always work reliably, so use chmodSync + // writeFileSync's mode option doesn't always work reliably, so use chmodSync chmodSync(sshKeyPath, 0o600); // If key has a passphrase, decrypt it in-place so SSH can use it non-interactively if (credential.sshPassphrase) { - const result = Bun.spawnSync([ - 'ssh-keygen', '-p', '-f', sshKeyPath, - '-P', credential.sshPassphrase, '-N', '' - ], { env, stderr: 'pipe' }); - if (result.exitCode !== 0) { + const result = spawnSync( + 'ssh-keygen', + ['-p', '-f', sshKeyPath, '-P', credential.sshPassphrase, '-N', ''], + { env, stdio: ['pipe', 'pipe', 'pipe'] } + ); + if (result.status !== 0) { const stderr = result.stderr.toString().trim(); console.warn(`[git] Failed to decrypt SSH key: ${stderr}`); } @@ -173,7 +198,7 @@ async function buildGitEnv(credential: GitCredential | null): Promise { function cleanupSshKey(credential: GitCredential | null): void { if (credential?.authType === 'ssh') { - const sshKeyPath = resolve(join(GIT_REPOS_DIR, `.ssh-key-${credential.id}`)); + const sshKeyPath = `/tmp/.ssh-key-${credential.id}`; try { if (existsSync(sshKeyPath)) { rmSync(sshKeyPath); @@ -207,21 +232,15 @@ function buildRepoUrl(url: string, credential: GitCredential | null): string { async function execGit(args: string[], cwd: string, env: GitEnv): Promise<{ stdout: string; stderr: string; code: number }> { try { - const proc = Bun.spawn(['git', ...args], { + const proc = nodeSpawn('git', args, { cwd, env, - stdout: 'pipe', - stderr: 'pipe' + stdio: ['pipe', 'pipe', 'pipe'] }); - const [stdout, stderr] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text() - ]); + const result = await collectProcess(proc); - const code = await proc.exited; - - return { stdout: stdout.trim(), stderr: stderr.trim(), code }; + return { stdout: result.stdout.trim(), stderr: result.stderr.trim(), code: result.exitCode }; } catch (err: any) { return { stdout: '', stderr: err.message, code: 1 }; } @@ -350,8 +369,6 @@ async function testRepositoryConnection(options: { env ); - cleanupSshKey(credential); - if (result.code !== 0) { console.error('[Git] Connection test failed:', result.stderr); return { success: false, error: cleanGitError(result.stderr) }; @@ -366,7 +383,6 @@ async function testRepositoryConnection(options: { process.cwd(), env ); - cleanupSshKey(credential); if (allBranchesResult.code !== 0) { return { success: false, error: cleanGitError(allBranchesResult.stderr) }; @@ -400,8 +416,9 @@ async function testRepositoryConnection(options: { lastCommit }; } catch (error: any) { - cleanupSshKey(credential); return { success: false, error: error.message }; + } finally { + cleanupSshKey(credential); } } @@ -519,7 +536,7 @@ export async function syncRepository(repoId: number): Promise { throw new Error(`Compose file not found: ${repo.composePath}`); } - const composeContent = await Bun.file(composePath).text(); + const composeContent = readFileSync(composePath, 'utf-8'); // Update repository status await updateGitRepository(repoId, { @@ -798,7 +815,7 @@ export async function syncGitStack(stackId: number): Promise { throw new Error(`Compose file not found: ${gitStack.composePath}`); } - const composeContent = await Bun.file(composePath).text(); + const composeContent = readFileSync(composePath, 'utf-8'); console.log(`${logPrefix} Compose content length:`, composeContent.length, 'chars'); console.log(`${logPrefix} Compose content:`); console.log(composeContent); @@ -819,7 +836,7 @@ export async function syncGitStack(stackId: number): Promise { if (existsSync(envFilePath)) { try { console.log(`${logPrefix} Reading env file...`); - envFileContent = await Bun.file(envFilePath).text(); + envFileContent = readFileSync(envFilePath, 'utf-8'); envFileVars = parseEnvFileContent(envFileContent, gitStack.stackName); console.log(`${logPrefix} Env file parsed, vars count:`, Object.keys(envFileVars).length); @@ -1142,7 +1159,7 @@ export async function deployGitStackWithProgress( throw new Error(`Compose file not found: ${gitStack.composePath}`); } - const composeContent = await Bun.file(composePath).text(); + const composeContent = readFileSync(composePath, 'utf-8'); // Determine the compose directory (for copying all files) const composeDir = dirname(composePath); @@ -1153,7 +1170,7 @@ export async function deployGitStackWithProgress( const envFilePath = join(repoPath, gitStack.envFilePath); if (existsSync(envFilePath)) { try { - const envContent = await Bun.file(envFilePath).text(); + const envContent = readFileSync(envFilePath, 'utf-8'); envFileVars = parseEnvFileContent(envContent, gitStack.stackName); } catch (err) { // Log but don't fail - env file is optional @@ -1251,12 +1268,11 @@ export async function listGitStackEnvFiles(stackId: number): Promise<{ files: st const maxDepth = 3; // Use find to locate all .env* files - const proc = Bun.spawn(['find', repoPath, '-maxdepth', String(maxDepth), '-type', 'f', '-name', '.env*'], { - stdout: 'pipe', - stderr: 'pipe' + const proc = nodeSpawn('find', [repoPath, '-maxdepth', String(maxDepth), '-type', 'f', '-name', '.env*'], { + stdio: ['pipe', 'pipe', 'pipe'] }); - const output = await new Response(proc.stdout).text(); - await proc.exited; + const findResult = await collectProcess(proc); + const output = findResult.stdout; const files = output.trim().split('\n').filter(f => f); const envFiles: string[] = []; @@ -1372,7 +1388,7 @@ export async function readGitStackEnvFile( } try { - const content = await Bun.file(fullPath).text(); + const content = readFileSync(fullPath, 'utf-8'); const vars = parseEnvFileContent(content); return { vars }; } catch (error: any) { @@ -1426,17 +1442,18 @@ export async function previewRepoEnvFiles(options: PreviewEnvOptions): Promise

; pendingStreamRequests: Map; lastMetrics?: { @@ -76,6 +77,9 @@ declare global { var __hawserSendMessage: ((envId: number, message: string) => boolean) | undefined; var __hawserHandleContainerEvent: ((envId: number, event: ContainerEventMessage['event']) => Promise) | undefined; var __hawserHandleMetrics: ((envId: number, metrics: MetricsMessage['metrics']) => Promise) | undefined; + var __hawserHandleMessage: ((ws: any, msg: any, connId: string) => Promise) | undefined; + var __hawserHandleDisconnect: ((ws: any, connId: string) => void) | undefined; + var __terminalHandleExecMessage: ((msg: any) => void) | undefined; } export const edgeConnections: Map = globalThis.__hawserEdgeConnections ?? (globalThis.__hawserEdgeConnections = new Map()); @@ -94,7 +98,7 @@ export function initializeEdgeManager(): void { const timeout = 90 * 1000; // 90 seconds (3 missed heartbeats) for (const [envId, conn] of edgeConnections) { - if (now - conn.lastHeartbeat.getTime() > timeout) { + if (now - conn.lastHeartbeat > timeout) { const pendingCount = conn.pendingRequests.size; const streamCount = conn.pendingStreamRequests.size; console.log( @@ -120,6 +124,21 @@ export function initializeEdgeManager(): void { updateEnvironmentStatus(envId, null); } } + + // Maintain reconnection tracker: reset for stable connections, prune stale entries + for (const [envId, tracker] of reconnectTracker) { + const conn = edgeConnections.get(envId); + if (conn && now - conn.lastHeartbeat < STABLE_THRESHOLD_MS) { + // Connection is stable — reset tracker so next reconnect is unthrottled + reconnectTracker.delete(envId); + } else if (!conn && tracker.timestamps.length > 0) { + const lastAttempt = tracker.timestamps[tracker.timestamps.length - 1]; + if (now - lastAttempt > STALE_TRACKER_MS) { + // No connection and no recent attempts — clean up + reconnectTracker.delete(envId); + } + } + } }, 30000); } @@ -137,6 +156,7 @@ export function stopEdgeManager(): void { conn.ws.close(1001, 'Server shutdown'); } edgeConnections.clear(); + reconnectTracker.clear(); } /** @@ -189,7 +209,7 @@ export async function handleEdgeContainerEvent( } } -// Register global handler for patch-build.ts to use +// Register global handler for server.js to use globalThis.__hawserHandleContainerEvent = handleEdgeContainerEvent; /** @@ -218,14 +238,8 @@ export async function handleEdgeMetrics( ? (metrics.memoryUsed / metrics.memoryTotal) * 100 : 0; - // Save to database using the existing function - await saveHostMetric( - cpuPercent, - memoryPercent, - metrics.memoryUsed, - metrics.memoryTotal, - environmentId - ); + // Push to in-memory ring buffer + pushMetric(environmentId, cpuPercent, memoryPercent, metrics.memoryUsed, metrics.memoryTotal); } catch (error) { const errorMsg = error instanceof Error ? error.message : String(error); console.error('[Hawser] Error saving metrics:', errorMsg); @@ -288,6 +302,13 @@ export async function generateHawserToken( edgeConnections.delete(environmentId); } + // Revoke all existing active tokens for this environment so the old agent + // can no longer reconnect and fight with the new one over the connection slot + await db + .update(hawserTokens) + .set({ isActive: false }) + .where(and(eq(hawserTokens.environmentId, environmentId), eq(hawserTokens.isActive, true))); + // Use provided token or generate a new one let token: string; if (rawToken) { @@ -396,6 +417,7 @@ export function handleEdgeConnection( // Reject all pending requests before closing for (const [requestId, pending] of existing.pendingRequests) { console.log(`[Hawser] Rejecting pending request ${requestId} due to connection replacement`); + clearTimeout(pending.timeout); pending.reject(new Error('Connection replaced by new agent')); } for (const [requestId, pending] of existing.pendingStreamRequests) { @@ -405,7 +427,12 @@ export function handleEdgeConnection( existing.pendingRequests.clear(); existing.pendingStreamRequests.clear(); - existing.ws.close(1000, 'Replaced by new connection'); + // Immediately destroy TCP socket — no graceful close needed for replaced connections + if (typeof existing.ws.terminate === 'function') { + existing.ws.terminate(); + } else { + existing.ws.close(1000, 'Replaced by new connection'); + } } const connection: EdgeConnection = { @@ -418,7 +445,7 @@ export function handleEdgeConnection( hostname: hello.hostname, capabilities: hello.capabilities, connectedAt: new Date(), - lastHeartbeat: new Date(), + lastHeartbeat: Date.now(), pendingRequests: new Map(), pendingStreamRequests: new Map() }; @@ -471,7 +498,8 @@ export async function sendEdgeRequest( body?: unknown, headers?: Record, streaming = false, - timeout = 30000 + timeout = 30000, + isBinary = false ): Promise { const connection = edgeConnections.get(environmentId); if (!connection) { @@ -559,7 +587,8 @@ export async function sendEdgeRequest( method, path, headers: headers || {}, - body: body, // Body is already an object, will be serialized by JSON.stringify(message) + body: body, + isBinary, // true when body is base64-encoded binary (tar uploads) streaming }; @@ -753,7 +782,7 @@ export function handleEdgeResponse(environmentId: number, response: ResponseMess export function handleHeartbeat(environmentId: number): void { const connection = edgeConnections.get(environmentId); if (connection) { - connection.lastHeartbeat = new Date(); + connection.lastHeartbeat = Date.now(); } } @@ -824,7 +853,8 @@ export interface RequestMessage { method: string; path: string; headers?: Record; - body?: unknown; // JSON-serializable object, will be serialized when message is stringified + body?: unknown; // JSON-serializable object, or base64 string when isBinary=true + isBinary?: boolean; // true when body is base64-encoded binary data (tar uploads) streaming?: boolean; } @@ -946,3 +976,269 @@ export type HawserMessage = | ContainerEventMessage | { type: 'ping'; timestamp: number } | { type: 'pong'; timestamp: number }; + +// ─── Production WebSocket message handler (used by server.js) ─── + +// Maps WebSocket instances to environment IDs for message routing +const wsToEnvId = new Map(); + +// Auth fail cache to prevent brute-force token validation. +// Entries are periodically cleaned up to prevent unbounded growth. +const hawserAuthFailCache = new Map(); +const HAWSER_AUTH_FAIL_COOLDOWN_MS = 30_000; + +// Periodic cleanup of expired auth fail entries (every 60s) +setInterval(() => { + const now = Date.now(); + for (const [key, timestamp] of hawserAuthFailCache) { + if (now - timestamp > HAWSER_AUTH_FAIL_COOLDOWN_MS) { + hawserAuthFailCache.delete(key); + } + } +}, 60_000); + +// ─── Reconnection storm throttle ─── +// Tracks per-environment reconnection frequency to detect storms +// (e.g., agent can auth but Docker socket is broken → 30s timeout → reconnect loop) +interface ReconnectTrackerEntry { + timestamps: number[]; + cooldownUntil: number; // 0 = no cooldown active + cooldownLevel: number; // index into COOLDOWN_LEVELS +} +const reconnectTracker = new Map(); +const RECONNECT_WINDOW_MS = 2 * 60 * 1000; // 2-minute sliding window +const RECONNECT_BURST = 3; // allow 3 reconnections per window +const COOLDOWN_LEVELS_SECS = [30, 60, 120, 300]; // escalating cooldown in seconds +const STABLE_THRESHOLD_MS = 5 * 60 * 1000; // stable connection resets tracker +const STALE_TRACKER_MS = 10 * 60 * 1000; // clean up stale tracker entries + +/** + * Record a reconnection for an environment and check if throttling is needed. + * Returns { allowed: true } or { allowed: false, retryAfter: seconds }. + */ +function recordReconnection(envId: number): { allowed: true } | { allowed: false; retryAfter: number } { + const now = Date.now(); + let entry = reconnectTracker.get(envId); + + if (!entry) { + entry = { timestamps: [now], cooldownUntil: 0, cooldownLevel: 0 }; + reconnectTracker.set(envId, entry); + return { allowed: true }; + } + + // Check if currently in cooldown + if (now < entry.cooldownUntil) { + const retryAfter = Math.ceil((entry.cooldownUntil - now) / 1000); + return { allowed: false, retryAfter }; + } + + // Prune timestamps outside the sliding window + entry.timestamps = entry.timestamps.filter(ts => now - ts < RECONNECT_WINDOW_MS); + entry.timestamps.push(now); + + // Check if burst limit exceeded + if (entry.timestamps.length > RECONNECT_BURST) { + const level = Math.min(entry.cooldownLevel, COOLDOWN_LEVELS_SECS.length - 1); + const cooldownSecs = COOLDOWN_LEVELS_SECS[level]; + entry.cooldownUntil = now + cooldownSecs * 1000; + entry.cooldownLevel = Math.min(entry.cooldownLevel + 1, COOLDOWN_LEVELS_SECS.length - 1); + + console.warn( + `[Hawser WS] Reconnection storm detected for env ${envId}: ` + + `${entry.timestamps.length} connections in ${RECONNECT_WINDOW_MS / 1000}s. ` + + `Cooldown ${cooldownSecs}s (level ${level})` + ); + + return { allowed: false, retryAfter: cooldownSecs }; + } + + return { allowed: true }; +} + +/** + * Handle a WebSocket message from a Hawser Edge agent. + * Full protocol handler: hello/welcome auth, ping/pong, + * response/stream routing, metrics, container events, exec sessions. + * + * Registered as globalThis.__hawserHandleMessage for server.js to call. + */ +async function handleHawserWsMessage(ws: any, msg: any, connId: string): Promise { + if (msg.type === 'hello') { + const remoteAddr = connId; + + // Rate limit auth failures + const lastFail = hawserAuthFailCache.get(remoteAddr); + if (lastFail && Date.now() - lastFail < HAWSER_AUTH_FAIL_COOLDOWN_MS) { + ws.send(JSON.stringify({ type: 'error', message: 'Too many failed attempts' })); + ws.close(1008, 'Rate limited'); + return; + } + + if (!msg.token) { + ws.send(JSON.stringify({ type: 'error', message: 'No token provided' })); + ws.close(1008, 'Missing token'); + return; + } + + try { + const result = await validateHawserToken(msg.token); + if (!result.valid || !result.environmentId) { + console.log(`[Hawser WS] Authentication failed for connection ${connId}`); + hawserAuthFailCache.set(remoteAddr, Date.now()); + ws.send(JSON.stringify({ type: 'error', message: 'Invalid token' })); + ws.close(1008, 'Invalid token'); + return; + } + + // Throttle reconnection storms (successful auth but broken Docker = rapid reconnect loop) + const throttle = recordReconnection(result.environmentId); + if (!throttle.allowed) { + console.log(`[Hawser WS] Throttling reconnection for env ${result.environmentId}: retry after ${throttle.retryAfter}s`); + ws.send(JSON.stringify({ + type: 'error', + message: `Reconnection throttled. Retry after ${throttle.retryAfter}s.`, + retryAfter: throttle.retryAfter + })); + ws.close(1008, 'Reconnection throttled'); + return; + } + + // Authenticated — register the connection + const connection = handleEdgeConnection(ws, result.environmentId, msg); + wsToEnvId.set(ws, result.environmentId); + + // Send welcome + ws.send(JSON.stringify({ + type: 'welcome', + serverId: 'dockhand', + version: HAWSER_PROTOCOL_VERSION + })); + + console.log(`[Hawser WS] Agent authenticated: env=${result.environmentId} agent=${msg.agentName || msg.agentId}`); + } catch (error: any) { + console.error('[Hawser WS] Auth error:', error.message); + ws.send(JSON.stringify({ type: 'error', message: 'Authentication failed' })); + ws.close(1011, 'Auth error'); + } + return; + } + + // All other messages require an authenticated connection + const envId = wsToEnvId.get(ws); + if (!envId) { + ws.send(JSON.stringify({ type: 'error', message: 'Not authenticated' })); + return; + } + + const connection = edgeConnections.get(envId); + if (!connection) return; + + // Update heartbeat + connection.lastHeartbeat = Date.now(); + + switch (msg.type) { + case 'ping': + ws.send(JSON.stringify({ type: 'pong', timestamp: Date.now() })); + break; + + case 'pong': + break; + + case 'response': { + const pending = connection.pendingRequests.get(msg.requestId); + if (pending) { + clearTimeout(pending.timeout); + connection.pendingRequests.delete(msg.requestId); + pending.resolve({ + statusCode: msg.statusCode, + headers: msg.headers || {}, + body: msg.body, + isBinary: msg.isBinary + }); + } + break; + } + + case 'stream': { + const streamPending = connection.pendingStreamRequests.get(msg.requestId); + if (streamPending) { + streamPending.onData?.(msg.data); + } + break; + } + + case 'stream_end': { + const streamReq = connection.pendingStreamRequests.get(msg.requestId); + if (streamReq) { + connection.pendingStreamRequests.delete(msg.requestId); + streamReq.onEnd?.(msg.reason); + } + break; + } + + case 'metrics': + if (globalThis.__hawserHandleMetrics) { + await globalThis.__hawserHandleMetrics(envId, msg.metrics); + } + break; + + case 'container_event': + if (globalThis.__hawserHandleContainerEvent) { + await globalThis.__hawserHandleContainerEvent(envId, msg.event); + } + break; + + case 'exec_ready': + case 'exec_output': + case 'exec_end': + // Forward exec messages to server.js/vite.config.ts via global callback + if (globalThis.__terminalHandleExecMessage) { + globalThis.__terminalHandleExecMessage(msg); + } + break; + + case 'error': + console.error(`[Hawser WS] Agent error (env ${envId}): ${msg.message}`); + // Forward exec-related errors (identified by requestId) to terminal handler + if (msg.requestId && globalThis.__terminalHandleExecMessage) { + globalThis.__terminalHandleExecMessage(msg); + } + break; + } +} + +/** + * Handle WebSocket disconnect for a Hawser Edge agent. + * Receives the actual ws object to correctly identify which connection closed. + */ +function handleHawserWsDisconnect(disconnectedWs: any, connId: string): void { + const envId = wsToEnvId.get(disconnectedWs); + if (!envId) { + // This ws was never authenticated (e.g., auth failed), nothing to clean up + return; + } + + const connection = edgeConnections.get(envId); + if (connection && connection.ws === disconnectedWs) { + console.log(`[Hawser WS] Agent disconnected: env=${envId}`); + + for (const [, pending] of connection.pendingRequests) { + clearTimeout(pending.timeout); + pending.reject(new Error('Agent disconnected')); + } + for (const [, pending] of connection.pendingStreamRequests) { + pending.onEnd?.('Agent disconnected'); + } + connection.pendingRequests.clear(); + connection.pendingStreamRequests.clear(); + + edgeConnections.delete(envId); + updateEnvironmentStatus(envId, null); + } + + wsToEnvId.delete(disconnectedWs); +} + +// Register global handlers for server.js to call +globalThis.__hawserHandleMessage = handleHawserWsMessage; +globalThis.__hawserHandleDisconnect = handleHawserWsDisconnect; diff --git a/src/lib/server/host-path.ts b/src/lib/server/host-path.ts index cdd4cfd..f66300c 100644 --- a/src/lib/server/host-path.ts +++ b/src/lib/server/host-path.ts @@ -20,6 +20,7 @@ */ import { readFileSync } from 'node:fs'; +import * as http from 'node:http'; import { resolve } from 'node:path'; // Cache the host data dir to avoid repeated API calls @@ -29,6 +30,11 @@ let detectionAttempted = false; // Cache ALL mounts for path translation (not just DATA_DIR) let cachedMounts: Array<{ source: string; destination: string }> | null = null; +// Cache Dockhand's own Docker access method (detected from container inspect) +// Used by scanner to replicate how Dockhand connects to Docker +let cachedOwnDockerHost: string | null = null; +let cachedOwnNetworkMode: string | null = null; + /** * Get our own container ID */ @@ -95,25 +101,48 @@ export async function detectHostDataDir(): Promise { try { // Query Docker API to inspect our own container + // Try unix socket first, fall back to TCP if DOCKER_HOST is set const socketPath = process.env.DOCKER_SOCKET || '/var/run/docker.sock'; + const dockerHost = process.env.DOCKER_HOST; - // Use fetch with unix socket - const response = await fetch(`http://localhost/containers/${containerId}/json`, { - // @ts-ignore - Bun supports unix sockets - unix: socketPath - }); + const containerInfo = await new Promise((resolvePromise, reject) => { + const reqOptions: http.RequestOptions = dockerHost?.startsWith('tcp://') + ? (() => { + const u = new URL(dockerHost.replace('tcp://', 'http://')); + return { hostname: u.hostname, port: u.port, path: `/containers/${containerId}/json`, method: 'GET' }; + })() + : { socketPath, path: `/containers/${containerId}/json`, method: 'GET' }; - if (!response.ok) { - console.warn(`[HostPath] Failed to inspect container: ${response.status}`); - return null; - } - - const containerInfo = await response.json() as { + const req = http.request(reqOptions, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => { + if (res.statusCode === 200) { + try { + resolvePromise(JSON.parse(Buffer.concat(chunks).toString('utf-8'))); + } catch { + reject(new Error('Failed to parse container inspect response')); + } + } else { + reject(new Error(`Container inspect failed: ${res.statusCode}`)); + } + }); + res.on('error', reject); + }); + req.on('error', reject); + req.end(); + }) as { Mounts?: Array<{ Type: string; Source: string; Destination: string; }>; + Config?: { + Env?: string[]; + }; + NetworkSettings?: { + Networks?: Record; + }; }; // Cache ALL mounts for later path translation (used by rewriteComposeVolumePaths) @@ -123,6 +152,30 @@ export async function detectHostDataDir(): Promise { })); console.log(`[HostPath] Cached ${cachedMounts.length} mount(s)`); + // Cache DOCKER_HOST from Dockhand's own env vars (if set) + // This tells us how Dockhand was configured to reach Docker + const envVars = containerInfo.Config?.Env || []; + for (const v of envVars) { + if (v.startsWith('DOCKER_HOST=')) { + cachedOwnDockerHost = v.substring('DOCKER_HOST='.length); + console.log(`[HostPath] Detected own DOCKER_HOST: ${cachedOwnDockerHost}`); + break; + } + } + + // Cache Dockhand's network (prefer non-default for service discovery) + const networks = containerInfo.NetworkSettings?.Networks; + if (networks) { + const custom = Object.keys(networks).filter( + n => n !== 'bridge' && n !== 'none' && n !== 'host' + ); + cachedOwnNetworkMode = custom.length > 0 ? custom[0] + : networks.bridge ? 'bridge' : null; + if (cachedOwnNetworkMode) { + console.log(`[HostPath] Detected own network: ${cachedOwnNetworkMode}`); + } + } + // Find the mount for our DATA_DIR const dataMount = containerInfo.Mounts?.find(m => m.Destination === dataDir); @@ -157,6 +210,25 @@ export function getHostDataDir(): string | null { return cachedHostDataDir; } +/** + * Get DOCKER_HOST from Dockhand's own container config (if set). + * Returns the TCP address (e.g., "tcp://socket-proxy:2375") or null. + * Populated by detectHostDataDir() at startup. + */ +export function getOwnDockerHost(): string | null { + return cachedOwnDockerHost; +} + +/** + * Get the Docker network Dockhand is attached to. + * Used to place scanner containers on the same network so they can reach + * TCP-based Docker endpoints (e.g., socket proxy). + * Populated by detectHostDataDir() at startup. + */ +export function getOwnNetworkMode(): string | null { + return cachedOwnNetworkMode; +} + /** * Translate a container path to host path * diff --git a/src/lib/server/jobs.ts b/src/lib/server/jobs.ts new file mode 100644 index 0000000..692bd75 --- /dev/null +++ b/src/lib/server/jobs.ts @@ -0,0 +1,63 @@ +import { randomUUID } from 'crypto'; + +export interface JobLine { + event?: string; // 'result', 'progress', etc. — undefined for bare data lines + data: unknown; +} + +export interface Job { + id: string; + status: 'running' | 'done' | 'error'; + lines: JobLine[]; + result?: unknown; + createdAt: number; + updatedAt: number; +} + +const jobs = new Map(); + +export function createJob(): Job { + const job: Job = { + id: randomUUID(), + status: 'running', + lines: [], + createdAt: Date.now(), + updatedAt: Date.now() + }; + jobs.set(job.id, job); + return job; +} + +export function getJob(id: string): Job | undefined { + return jobs.get(id); +} + +export function appendLine(job: Job, line: JobLine): void { + job.lines.push(line); + job.updatedAt = Date.now(); +} + +export function completeJob(job: Job, result: unknown): void { + job.result = result; + job.status = 'done'; + job.updatedAt = Date.now(); +} + +export function failJob(job: Job, error: string): void { + job.result = { success: false, error }; + job.status = 'error'; + job.updatedAt = Date.now(); +} + +// Cleanup jobs older than 10 minutes that are no longer running +const CLEANUP_INTERVAL_MS = 60_000; +const JOB_TTL_MS = 10 * 60_000; + +setInterval(() => { + const cutoff = Date.now() - JOB_TTL_MS; + for (const [id, job] of jobs) { + if (job.status !== 'running' && job.updatedAt < cutoff) { + jobs.delete(id); + } + } +}, CLEANUP_INTERVAL_MS); diff --git a/src/lib/server/metrics-store.ts b/src/lib/server/metrics-store.ts new file mode 100644 index 0000000..848a8d6 --- /dev/null +++ b/src/lib/server/metrics-store.ts @@ -0,0 +1,124 @@ +/** + * In-Memory Metrics Ring Buffer + * + * Replaces SQLite/PostgreSQL host_metrics storage with a fixed-size + * in-memory circular buffer per environment. Uses pre-allocated arrays + * with head/count indices to avoid splice()-based eviction which causes + * V8 to repeatedly reallocate backing arrays. + * + * Memory: 16 envs × 360 slots × ~100 bytes ≈ 576 KB + */ + +export interface MetricPoint { + id: number; + cpuPercent: number; + memoryPercent: number; + memoryUsed: number; + memoryTotal: number; + environmentId: number | null; + timestamp: string; +} + +const MAX_POINTS_PER_ENV = 360; // 1 hour at 10s interval, 3 hours at 30s + +interface RingBuffer { + data: (MetricPoint | null)[]; + head: number; // next write position + count: number; // number of valid entries (≤ MAX_POINTS_PER_ENV) +} + +// envId → RingBuffer +const store = new Map(); + +let nextId = 1; + +/** + * Push a new metric data point for an environment. + * Overwrites oldest entry when buffer is full (no array reallocation). + */ +export function pushMetric( + envId: number, + cpuPercent: number, + memoryPercent: number, + memoryUsed: number, + memoryTotal: number +): void { + let ring = store.get(envId); + if (!ring) { + ring = { data: new Array(MAX_POINTS_PER_ENV).fill(null), head: 0, count: 0 }; + store.set(envId, ring); + } + + ring.data[ring.head] = { + id: nextId++, + cpuPercent, + memoryPercent, + memoryUsed, + memoryTotal, + environmentId: envId, + timestamp: new Date().toISOString() + }; + ring.head = (ring.head + 1) % MAX_POINTS_PER_ENV; + if (ring.count < MAX_POINTS_PER_ENV) ring.count++; +} + +/** + * Read entries from a ring buffer in oldest-first order. + */ +function readRing(ring: RingBuffer, limit: number): MetricPoint[] { + const count = Math.min(ring.count, limit); + if (count === 0) return []; + + const result: MetricPoint[] = new Array(count); + // Start reading from the oldest entry + const start = (ring.head - ring.count + MAX_POINTS_PER_ENV) % MAX_POINTS_PER_ENV; + const skip = ring.count - count; + const readFrom = (start + skip) % MAX_POINTS_PER_ENV; + + for (let i = 0; i < count; i++) { + result[i] = ring.data[(readFrom + i) % MAX_POINTS_PER_ENV]!; + } + return result; +} + +/** + * Get the most recent metric for an environment. + */ +export function getLatestMetric(envId: number): MetricPoint | null { + const ring = store.get(envId); + if (!ring || ring.count === 0) return null; + // head points to next write position, so latest is head - 1 + const idx = (ring.head - 1 + MAX_POINTS_PER_ENV) % MAX_POINTS_PER_ENV; + return ring.data[idx]; +} + +/** + * Get metrics history for an environment, oldest first. + */ +export function getMetricsHistory(envId: number, limit = 60): MetricPoint[] { + const ring = store.get(envId); + if (!ring || ring.count === 0) return []; + return readRing(ring, limit); +} + +/** + * Get all metrics (across all environments), newest first, with optional limit. + * Used by the global getHostMetrics() fallback when no envId is specified. + */ +export function getAllMetrics(limit = 60): MetricPoint[] { + const all: MetricPoint[] = []; + for (const ring of store.values()) { + const points = readRing(ring, ring.count); + all.push(...points); + } + // Sort newest first (matching old DB query behavior) + all.sort((a, b) => b.id - a.id); + return all.slice(0, limit); +} + +/** + * Clear all metrics for an environment (e.g., when environment is deleted). + */ +export function clearEnvironmentMetrics(envId: number): void { + store.delete(envId); +} diff --git a/src/lib/server/notifications.ts b/src/lib/server/notifications.ts index 01936d5..2dc4989 100644 --- a/src/lib/server/notifications.ts +++ b/src/lib/server/notifications.ts @@ -21,6 +21,13 @@ function escapeTelegramMarkdown(text: string): string { .replace(/`/g, '\\`'); // Backtick (code) } +/** Drain a response body to release the underlying socket/TLS connection. */ +async function drainResponse(response: Response): Promise { + if (!response.bodyUsed) { + try { await response.arrayBuffer(); } catch {} + } +} + export interface NotificationPayload { title: string; message: string; @@ -35,21 +42,47 @@ export interface NotificationResult { error?: string; } +// SMTP transporter cache — reuses connections instead of creating a new TLS pool per notification. +const transporterCache = new Map; lastUsed: number }>(); + +function getOrCreateTransporter(config: SmtpConfig): ReturnType { + const key = `${config.host}:${config.port}:${config.secure}:${config.username || ''}`; + const cached = transporterCache.get(key); + if (cached) { + cached.lastUsed = Date.now(); + return cached.transporter; + } + const transporter = nodemailer.createTransport({ + host: config.host, + port: config.port, + secure: config.secure, + auth: config.username ? { + user: config.username, + pass: config.password + } : undefined, + tls: config.skipTlsVerify ? { + rejectUnauthorized: false + } : undefined + }); + transporterCache.set(key, { transporter, lastUsed: Date.now() }); + return transporter; +} + +// Clean up idle transporters every 10 minutes +setInterval(() => { + const now = Date.now(); + for (const [key, entry] of transporterCache) { + if (now - entry.lastUsed > 10 * 60 * 1000) { + entry.transporter.close(); + transporterCache.delete(key); + } + } +}, 10 * 60 * 1000); + // Send notification via SMTP async function sendSmtpNotification(config: SmtpConfig, payload: NotificationPayload): Promise { try { - const transporter = nodemailer.createTransport({ - host: config.host, - port: config.port, - secure: config.secure, - auth: config.username ? { - user: config.username, - pass: config.password - } : undefined, - tls: config.skipTlsVerify ? { - rejectUnauthorized: false - } : undefined - }); + const transporter = getOrCreateTransporter(config); const envBadge = payload.environmentName ? `${payload.environmentName}` @@ -172,6 +205,7 @@ async function sendDiscord(appriseUrl: string, payload: NotificationPayload): Pr const text = await response.text().catch(() => ''); return { success: false, error: `Discord error ${response.status}: ${text || response.statusText}` }; } + await drainResponse(response); return { success: true }; } catch (error) { return { success: false, error: `Discord connection failed: ${error instanceof Error ? error.message : String(error)}` }; @@ -204,6 +238,7 @@ async function sendSlack(appriseUrl: string, payload: NotificationPayload): Prom const text = await response.text().catch(() => ''); return { success: false, error: `Slack error ${response.status}: ${text || response.statusText}` }; } + await drainResponse(response); return { success: true }; } catch (error) { return { success: false, error: `Slack connection failed: ${error instanceof Error ? error.message : String(error)}` }; @@ -211,7 +246,7 @@ async function sendSlack(appriseUrl: string, payload: NotificationPayload): Prom } // Mattermost webhook -async function sendMattermost(appriseUrl: string, payload: NotificationPayload): Promise { +async function sendMattermost(appriseUrl: string, payload: NotificationPayload): Promise { // mmost://[botname@]hostname[:port][/path]/token or mmosts://... const isSecure = appriseUrl.startsWith('mmosts'); const protocol = isSecure ? 'https' : 'http'; @@ -230,8 +265,7 @@ async function sendMattermost(appriseUrl: string, payload: NotificationPayload): // The token is the last segment, everything else is hostname[:port][/path] const lastSlashIndex = urlPart.lastIndexOf('/'); if (lastSlashIndex === -1) { - console.error('[Notifications] Invalid Mattermost URL format. Expected: mmost://[botname@]hostname[:port][/path]/token'); - return false; + return { success: false, error: 'Invalid Mattermost URL format. Expected: mmost://[botname@]hostname[:port][/path]/token' }; } const token = urlPart.substring(lastSlashIndex + 1); @@ -249,13 +283,22 @@ async function sendMattermost(appriseUrl: string, payload: NotificationPayload): body.username = username; } - const response = await fetch(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body) - }); + try { + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); - return response.ok; + if (!response.ok) { + const text = await response.text().catch(() => ''); + return { success: false, error: `Mattermost error ${response.status}: ${text || response.statusText}` }; + } + await drainResponse(response); + return { success: true }; + } catch (error) { + return { success: false, error: `Mattermost connection failed: ${error instanceof Error ? error.message : String(error)}` }; + } } // Telegram @@ -290,6 +333,7 @@ async function sendTelegram(appriseUrl: string, payload: NotificationPayload): P const errorMsg = errorData.description || response.statusText; return { success: false, error: `Telegram error ${response.status}: ${errorMsg}` }; } + await drainResponse(response); return { success: true }; } catch (error) { return { success: false, error: `Telegram connection failed: ${error instanceof Error ? error.message : String(error)}` }; @@ -328,6 +372,7 @@ async function sendGotify(appriseUrl: string, payload: NotificationPayload): Pro const text = await response.text().catch(() => ''); return { success: false, error: `Gotify error ${response.status}: ${text || response.statusText}` }; } + await drainResponse(response); return { success: true }; } catch (error) { return { success: false, error: `Gotify connection failed: ${error instanceof Error ? error.message : String(error)}` }; @@ -394,6 +439,7 @@ async function sendNtfy(appriseUrl: string, payload: NotificationPayload): Promi const text = await response.text().catch(() => ''); return { success: false, error: `ntfy error ${response.status}: ${text || response.statusText}` }; } + await drainResponse(response); return { success: true }; } catch (error) { return { success: false, error: `ntfy connection failed: ${error instanceof Error ? error.message : String(error)}` }; @@ -428,6 +474,7 @@ async function sendPushover(appriseUrl: string, payload: NotificationPayload): P const text = await response.text().catch(() => ''); return { success: false, error: `Pushover error ${response.status}: ${text || response.statusText}` }; } + await drainResponse(response); return { success: true }; } catch (error) { return { success: false, error: `Pushover connection failed: ${error instanceof Error ? error.message : String(error)}` }; @@ -455,6 +502,7 @@ async function sendGenericWebhook(appriseUrl: string, payload: NotificationPaylo const text = await response.text().catch(() => ''); return { success: false, error: `Webhook error ${response.status}: ${text || response.statusText}` }; } + await drainResponse(response); return { success: true }; } catch (error) { return { success: false, error: `Webhook connection failed: ${error instanceof Error ? error.message : String(error)}` }; diff --git a/src/lib/server/rss-tracker.ts b/src/lib/server/rss-tracker.ts new file mode 100644 index 0000000..d41fd10 --- /dev/null +++ b/src/lib/server/rss-tracker.ts @@ -0,0 +1,325 @@ +/** + * RSS Tracker — Per-operation native memory delta tracking + * + * Measures process.memoryUsage().rss before and after instrumented operations + * to identify which background operation is responsible for native memory growth. + * + * All functions are no-ops when MEMORY_MONITOR !== 'true'. + */ + +import v8 from 'node:v8'; +import { existsSync, mkdirSync, readdirSync, unlinkSync, statSync } from 'node:fs'; +import { join } from 'node:path'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface CategoryStats { + count: number; + totalDelta: number; + maxDelta: number; + // Lifetime cumulative (never reset) + lifetimeCount: number; + lifetimeTotalDelta: number; +} + +interface RssSnapshot { + filename: string; + timestamp: string; + uptimeMin: number; + rssMB: number; + sizeMB: number; +} + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +const enabled = process.env.MEMORY_MONITOR === 'true'; + +const categories = new Map(); +let intervalHandle: ReturnType | null = null; +let snapshotIntervalHandle: ReturnType | null = null; +let periodNumber = 0; +let periodStartRss = 0; +const startupTime = Date.now(); +const startupRss = enabled ? process.memoryUsage().rss : 0; + +// Snapshot settings +const SNAPSHOT_WARMUP_MS = 5 * 60 * 1000; // 5 min before first snapshot +const SNAPSHOT_INTERVAL_MS = parseInt(process.env.SNAPSHOT_INTERVAL || '60', 10) * 60 * 1000; +const MAX_SNAPSHOTS = 48; + +// --------------------------------------------------------------------------- +// Core API +// --------------------------------------------------------------------------- + +/** + * Capture RSS before an operation. Returns the RSS value. + * No-op (returns 0) when MEMORY_MONITOR is not set. + */ +export function rssBeforeOp(): number { + if (!enabled) return 0; + return process.memoryUsage().rss; +} + +/** + * Record RSS delta after an operation. + * No-op when MEMORY_MONITOR is not set. + */ +export function rssAfterOp(category: string, before: number): void { + if (!enabled || before === 0) return; + + const after = process.memoryUsage().rss; + const delta = after - before; + + let stats = categories.get(category); + if (!stats) { + stats = { count: 0, totalDelta: 0, maxDelta: 0, lifetimeCount: 0, lifetimeTotalDelta: 0 }; + categories.set(category, stats); + } + + stats.count++; + stats.totalDelta += delta; + if (Math.abs(delta) > Math.abs(stats.maxDelta)) stats.maxDelta = delta; + + stats.lifetimeCount++; + stats.lifetimeTotalDelta += delta; +} + +/** + * Get current stats summary. + */ +export function getRssStats() { + if (!enabled) return null; + + const mem = process.memoryUsage(); + const uptimeMs = Date.now() - startupTime; + const uptimeHours = uptimeMs / (1000 * 60 * 60); + const rssGrowth = mem.rss - startupRss; + + const perCategory: Record = {}; + + for (const [cat, stats] of categories) { + perCategory[cat] = { + count: stats.count, + avgDelta: stats.count > 0 ? fmtBytes(Math.round(stats.totalDelta / stats.count)) : '0', + maxDelta: fmtBytes(stats.maxDelta), + totalDelta: fmtBytes(stats.totalDelta), + lifetimeCount: stats.lifetimeCount, + lifetimeTotalDelta: fmtBytes(stats.lifetimeTotalDelta), + }; + } + + return { + enabled: true, + periodNumber, + rssMB: fmtMB(mem.rss), + rssGrowthTotal: fmtBytes(rssGrowth), + rssGrowthPerHour: fmtBytes(uptimeHours > 0.01 ? rssGrowth / uptimeHours : 0), + uptimeHours: Math.round(uptimeHours * 100) / 100, + categories: perCategory, + }; +} + +// --------------------------------------------------------------------------- +// Periodic logging +// --------------------------------------------------------------------------- + +function logPeriodSummary(): void { + periodNumber++; + const mem = process.memoryUsage(); + const rssDelta = mem.rss - periodStartRss; + const uptimeMs = Date.now() - startupTime; + const uptimeHours = uptimeMs / (1000 * 60 * 60); + const rssGrowthTotal = mem.rss - startupRss; + const rssPerHour = uptimeHours > 0.01 ? rssGrowthTotal / uptimeHours : 0; + + let summary = `[RSS] #${periodNumber} rss=${fmtMB(mem.rss)}(${fmtDelta(rssDelta)}) total=${fmtDelta(rssGrowthTotal)} rate=${fmtBytes(Math.round(rssPerHour))}/h`; + + // Sort categories by absolute totalDelta descending + const sorted = [...categories.entries()] + .filter(([, s]) => s.count > 0) + .sort((a, b) => Math.abs(b[1].totalDelta) - Math.abs(a[1].totalDelta)); + + let accountedDelta = 0; + for (const [cat, stats] of sorted) { + const avg = stats.count > 0 ? Math.round(stats.totalDelta / stats.count) : 0; + summary += `\n ${cat.padEnd(14)} n=${String(stats.count).padStart(4)} avg=${fmtDelta(avg).padStart(7)} max=${fmtDelta(stats.maxDelta).padStart(7)} total=${fmtDelta(stats.totalDelta).padStart(7)}`; + accountedDelta += stats.totalDelta; + } + + const unaccounted = rssDelta - accountedDelta; + if (sorted.length > 0) { + summary += `\n ${'unaccounted'.padEnd(14)} ${fmtDelta(unaccounted).padStart(7)}`; + } + + console.log(summary); + + // Reset per-period counters (keep lifetime) + for (const stats of categories.values()) { + stats.count = 0; + stats.totalDelta = 0; + stats.maxDelta = 0; + } + periodStartRss = mem.rss; +} + +// --------------------------------------------------------------------------- +// Heap snapshots +// --------------------------------------------------------------------------- + +function getSnapshotDir(): string { + const dataDir = process.env.DATA_DIR || './data'; + return join(dataDir, 'snapshots'); +} + +function ensureSnapshotDir(): string { + const dir = getSnapshotDir(); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + return dir; +} + +function cleanupOldSnapshots(dir: string): void { + try { + const files = readdirSync(dir) + .filter(f => f.endsWith('.heapsnapshot')) + .map(f => ({ name: f, time: statSync(join(dir, f)).mtimeMs })) + .sort((a, b) => a.time - b.time); + + while (files.length > MAX_SNAPSHOTS) { + const oldest = files.shift()!; + try { + unlinkSync(join(dir, oldest.name)); + } catch { /* ignore */ } + } + } catch { /* ignore */ } +} + +/** + * Dump a V8 heap snapshot to disk. Returns the filename. + */ +export function dumpHeapSnapshot(): string | null { + if (!enabled) return null; + + const dir = ensureSnapshotDir(); + cleanupOldSnapshots(dir); + + const mem = process.memoryUsage(); + const uptimeMin = Math.round((Date.now() - startupTime) / 60000); + const rssMB = Math.round(mem.rss / (1024 * 1024)); + const ts = new Date().toISOString().replace(/[:.]/g, '-').replace('T', '-').substring(0, 19); + const filename = `heap-${ts}-${uptimeMin}m-${rssMB}mb.heapsnapshot`; + const filepath = join(dir, filename); + + try { + v8.writeHeapSnapshot(filepath); + console.log(`[RSS] Heap snapshot saved: ${filepath}`); + return filename; + } catch (err) { + console.error(`[RSS] Failed to write heap snapshot:`, err instanceof Error ? err.message : String(err)); + return null; + } +} + +/** + * List saved heap snapshots. + */ +export function listHeapSnapshots(): RssSnapshot[] { + const dir = getSnapshotDir(); + if (!existsSync(dir)) return []; + + try { + return readdirSync(dir) + .filter(f => f.endsWith('.heapsnapshot')) + .map(f => { + const stat = statSync(join(dir, f)); + // Parse filename: heap-YYYY-MM-DD-HH-MM-SS-{uptime}m-{rss}mb.heapsnapshot + const match = f.match(/heap-(.+?)-(\d+)m-(\d+)mb\.heapsnapshot/); + return { + filename: f, + timestamp: stat.mtime.toISOString(), + uptimeMin: match ? parseInt(match[2]) : 0, + rssMB: match ? parseInt(match[3]) : 0, + sizeMB: Math.round(stat.size / (1024 * 1024) * 10) / 10, + }; + }) + .sort((a, b) => a.timestamp.localeCompare(b.timestamp)); + } catch { + return []; + } +} + +function startAutoSnapshots(): void { + // First snapshot after warmup + setTimeout(() => { + dumpHeapSnapshot(); + + // Then every SNAPSHOT_INTERVAL_MS + snapshotIntervalHandle = setInterval(() => { + dumpHeapSnapshot(); + }, SNAPSHOT_INTERVAL_MS); + }, SNAPSHOT_WARMUP_MS); +} + +// --------------------------------------------------------------------------- +// Lifecycle +// --------------------------------------------------------------------------- + +/** + * Start the RSS tracker. Call once on startup. + */ +export function startRssTracker(): void { + if (!enabled) return; + + periodStartRss = process.memoryUsage().rss; + console.log(`[RSS] Tracker started. Initial RSS: ${fmtMB(periodStartRss)}. Logging every 60s.`); + + intervalHandle = setInterval(logPeriodSummary, 60_000); + + startAutoSnapshots(); +} + +/** + * Stop the RSS tracker. + */ +export function stopRssTracker(): void { + if (intervalHandle) { + clearInterval(intervalHandle); + intervalHandle = null; + } + if (snapshotIntervalHandle) { + clearInterval(snapshotIntervalHandle); + snapshotIntervalHandle = null; + } +} + +// --------------------------------------------------------------------------- +// Formatting helpers +// --------------------------------------------------------------------------- + +function fmtMB(bytes: number): string { + return `${(bytes / (1024 * 1024)).toFixed(1)}M`; +} + +function fmtBytes(bytes: number): string { + const abs = Math.abs(bytes); + const sign = bytes < 0 ? '-' : '+'; + if (abs < 1024) return `${sign}${abs}B`; + if (abs < 1024 * 1024) return `${sign}${(abs / 1024).toFixed(1)}K`; + return `${sign}${(abs / (1024 * 1024)).toFixed(1)}M`; +} + +function fmtDelta(bytes: number): string { + return fmtBytes(bytes); +} diff --git a/src/lib/server/scanner.ts b/src/lib/server/scanner.ts index 505e9ac..080b9cc 100644 --- a/src/lib/server/scanner.ts +++ b/src/lib/server/scanner.ts @@ -15,7 +15,7 @@ import { } from './docker'; import { getEnvironment, getEnvSetting, getSetting } from './db'; import { sendEventNotification } from './notifications'; -import { getHostDockerSocket, getHostDataDir, extractUidFromSocketPath } from './host-path'; +import { getHostDockerSocket, getHostDataDir, extractUidFromSocketPath, getOwnDockerHost, getOwnNetworkMode } from './host-path'; import { resolve } from 'node:path'; import { mkdir, chown } from 'node:fs/promises'; @@ -306,8 +306,18 @@ export function sanitizeJsonString(json: string): string { const ch2 = json[i]; if ('"\\\/bfnrtu'.includes(ch2)) { result += ch2; + } else if (ch < 0x20) { + // Backslash followed by a raw control character (e.g. \ + 0x0A) + // The backslash was already added to result — escape it as \\ + // then also escape the control character + result += '\\'; + if (ch === 0x0A) result += '\\n'; + else if (ch === 0x0D) result += '\\r'; + else if (ch === 0x09) result += '\\t'; + else result += `\\u${ch.toString(16).padStart(4, '0')}`; + sanitized++; } else { - // Invalid escape like \x, \a, \0, \_ — convert backslash to literal \\ + // Invalid escape like \x, \a, \e — convert backslash to literal \\ result += '\\' + ch2; sanitized++; } @@ -341,7 +351,7 @@ export function sanitizeJsonString(json: string): string { } if (sanitized > 0) { - console.warn(`[Scanner] Sanitized ${sanitized} control characters in JSON output`); + console.warn(`[Scanner] Sanitized ${sanitized} control/escape characters in JSON output`); } return result; @@ -359,12 +369,10 @@ function parseGrypeOutput(output: string): { vulnerabilities: Vulnerability[]; s unknown: 0 }; - console.log('[Grype] Raw output length:', output.length); - console.log('[Grype] Output starts with:', output.slice(0, 200)); - console.log('[Grype] Output ends with:', JSON.stringify(output.slice(-50))); - try { - const data = JSON.parse(sanitizeJsonString(extractJson(output))); + const extracted = extractJson(output); + const sanitized = sanitizeJsonString(extracted); + const data = JSON.parse(sanitized); if (data.matches) { for (const match of data.matches) { @@ -587,42 +595,45 @@ async function runScannerContainerCore( const basePath = scannerType === 'grype' ? '/cache/grype' : '/cache/trivy'; const dbPath = basePath; - // Detect the host Docker socket path based on connection type - // For local socket environments, detect the actual host socket path (handles rootless Docker) - // For remote environments (hawser/direct with host), scanner runs remotely and uses standard path + // Detect how the scanner container should access Docker. + // Strategy: mirror Dockhand's own Docker connection when running locally. const env = envId ? await getEnvironment(envId) : undefined; const connectionType = env?.connectionType; - // Determine if this is a local socket environment: - // - connectionType === 'socket' (explicit) - // - connectionType is null/undefined (default behavior) - // - connectionType === 'direct' but no host specified (legacy local environments) - const isLocalSocket = !connectionType || - connectionType === 'socket' || - (connectionType === 'direct' && !env?.host); + const isHawser = connectionType === 'hawser-standard' || connectionType === 'hawser-edge'; - let hostSocketPath: string; + let hostSocketPath: string | null = null; let rootlessUid: string | undefined; + let scannerNetworkMode: string | undefined; + let scannerDockerHost: string | undefined; - if (isLocalSocket) { - // Local socket environment - detect host socket path (handles rootless Docker) + // Check if Dockhand itself uses TCP to reach Docker (e.g., socket proxy). + // Detected at startup from Dockhand's own container inspect data. + // This applies to ALL non-hawser environments since the scanner container + // runs on the same Docker daemon and needs the same access method. + const ownDockerHost = getOwnDockerHost(); + + if (!isHawser && ownDockerHost?.startsWith('tcp://')) { + // TCP mode: scanner uses the same DOCKER_HOST + network as Dockhand + scannerDockerHost = ownDockerHost; + scannerNetworkMode = getOwnNetworkMode() ?? undefined; + console.log(`[Scanner] TCP mode (from container inspect) - DOCKER_HOST=${scannerDockerHost}, network=${scannerNetworkMode ?? 'default'}`); + } else if (isHawser) { + // Hawser: scanner runs on remote host, uses remote host's standard Docker socket + hostSocketPath = '/var/run/docker.sock'; + console.log(`[Scanner] Remote scan via Hawser (${connectionType}) - using standard socket path`); + } else { + // Local socket — detect host socket path (handles rootless Docker) hostSocketPath = getHostDockerSocket(); console.log(`[Scanner] Local socket scan (${connectionType || 'default'}) - detected host Docker socket: ${hostSocketPath}`); // For user-specific Docker sockets (rootless Docker), detect UID for cache ownership - // but do NOT set container user — in rootless Docker, root inside the container - // maps to the socket-owning UID on the host via user namespace remapping const uid = extractUidFromSocketPath(hostSocketPath); if (uid) { rootlessUid = uid; console.log(`[Scanner] Rootless Docker detected (UID ${rootlessUid})`); console.log(`[Scanner] Scanner will run as root inside container (maps to UID ${rootlessUid} on host via user namespace)`); } - } else { - // Remote environment (direct with host/hawser-standard/hawser-edge) - // Scanner runs on remote host, uses remote host's standard Docker socket - hostSocketPath = '/var/run/docker.sock'; - console.log(`[Scanner] Remote scan (${connectionType}, host: ${env?.host}) - using standard socket path: ${hostSocketPath}`); } // Determine cache storage strategy based on environment @@ -643,10 +654,12 @@ async function runScannerContainerCore( console.log(`[Scanner] Standard mode - using volume: ${volumeName}`); } - const binds = [ - `${hostSocketPath}:/var/run/docker.sock:ro`, - cacheBind - ]; + // Build binds — only include socket mount when using socket mode + const binds: string[] = []; + if (hostSocketPath) { + binds.push(`${hostSocketPath}:/var/run/docker.sock:ro`); + } + binds.push(cacheBind); console.log(`[Scanner] Container bind mounts: ${JSON.stringify(binds)}`); @@ -655,6 +668,11 @@ async function runScannerContainerCore( ? [`GRYPE_DB_CACHE_DIR=${dbPath}`] : [`TRIVY_CACHE_DIR=${dbPath}`]; + // In TCP mode, pass DOCKER_HOST so scanner connects to Docker via TCP + if (scannerDockerHost) { + envVars.push(`DOCKER_HOST=${scannerDockerHost}`); + } + console.log(`[Scanner] Running ${scannerType} with cache mounted at ${basePath}`); console.log(`[Scanner] Container command: ${cmd.join(' ')}`); // Run the scanner container with a 10-minute timeout to prevent indefinite hangs @@ -665,6 +683,7 @@ async function runScannerContainerCore( env: envVars, name: `dockhand-${scannerType}-${Date.now()}`, envId, + networkMode: scannerNetworkMode, timeout: 600_000, // 10 minutes onStderr: (data) => { // Stream stderr lines for real-time progress output @@ -680,8 +699,8 @@ async function runScannerContainerCore( console.log(`[Scanner] ${scannerType} container completed, output length: ${output.length}`); if (output.length === 0) { console.error(`[Scanner] WARNING: Empty output from ${scannerType} container`); - console.error(`[Scanner] This may indicate the scanner couldn't access Docker socket`); - console.error(`[Scanner] Host socket path used: ${hostSocketPath}`); + console.error(`[Scanner] This may indicate the scanner couldn't access Docker`); + console.error(`[Scanner] Docker access: ${scannerDockerHost ? `TCP ${scannerDockerHost}` : `socket ${hostSocketPath}`}`); } else if (output.length < 100) { console.log(`[Scanner] ${scannerType} output preview: ${output}`); } diff --git a/src/lib/server/scheduler/tasks/container-update.ts b/src/lib/server/scheduler/tasks/container-update.ts index 892f1ad..22bcc4c 100644 --- a/src/lib/server/scheduler/tasks/container-update.ts +++ b/src/lib/server/scheduler/tasks/container-update.ts @@ -569,9 +569,9 @@ export async function runContainerUpdate( // ============================================================================= log(`Recreating container with full config passthrough...`); - const success = await recreateContainer(containerName, envId, log, imageNameFromConfig); + const result = await recreateContainer(containerName, envId, log, imageNameFromConfig); - if (success) { + if (result.success) { await updateAutoUpdateLastUpdated(containerName, envId); log(`Successfully updated container: ${containerName}`); @@ -594,7 +594,7 @@ export async function runContainerUpdate( type: 'success' }, envId); } else { - throw new Error('Failed to recreate container'); + throw new Error(result.error || 'Failed to recreate container'); } } catch (error: any) { @@ -628,14 +628,14 @@ export async function recreateContainer( envId?: number, log?: (msg: string) => void, imageNameOverride?: string -): Promise { +): Promise<{ success: boolean; error?: string }> { try { const containers = await listContainers(true, envId); const container = containers.find(c => c.name === containerName); if (!container) { log?.(`Container not found: ${containerName}`); - return false; + return { success: false, error: `Container not found: ${containerName}` }; } const inspectData = await inspectContainer(container.id, envId) as any; @@ -644,10 +644,10 @@ export async function recreateContainer( log?.(`Recreating container: ${containerName} (image: ${imageName})`); await recreateContainerFromInspect(inspectData, imageName, envId, log); - return true; + return { success: true }; } catch (error: any) { log?.(`Failed to recreate container: ${error.message}`); - return false; + return { success: false, error: error.message }; } } diff --git a/src/lib/server/scheduler/tasks/env-update-check.ts b/src/lib/server/scheduler/tasks/env-update-check.ts index 38b6289..f8bca9b 100644 --- a/src/lib/server/scheduler/tasks/env-update-check.ts +++ b/src/lib/server/scheduler/tasks/env-update-check.ts @@ -352,9 +352,9 @@ export async function runEnvUpdateCheckJob( // Recreate container with full config passthrough await log(` Recreating container...`); - const ok = await recreateContainer(update.containerName, environmentId, + const result = await recreateContainer(update.containerName, environmentId, (msg) => { log(` ${msg}`); }); - if (!ok) throw new Error('Container recreation failed'); + if (!result.success) throw new Error(result.error || 'Container recreation failed'); await log(` Updated successfully`); successCount++; diff --git a/src/lib/server/sse.ts b/src/lib/server/sse.ts new file mode 100644 index 0000000..6397b53 --- /dev/null +++ b/src/lib/server/sse.ts @@ -0,0 +1,145 @@ +import { json } from '@sveltejs/kit'; +import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs'; + +/** + * Check if the client prefers JSON over SSE. + * Returns true when Accept header includes application/json but NOT text/event-stream. + */ +export function prefersJSON(request?: Request): boolean { + const accept = request?.headers.get('accept') || ''; + return accept.includes('application/json') && !accept.includes('text/event-stream'); +} + +/** + * Wrap an SSE Response for JSON-preferring clients. + * + * Consumes the SSE stream using proper event framing (blank-line delimited, + * multi-line data joined with \n, CRLF stripped). Returns the `result` event + * data as a JSON response, or a fallback if no result event was emitted. + * + * Usage: + * if (prefersJSON(request)) return sseToJSON(buildSSEResponse()); + * return buildSSEResponse(); + */ +export async function sseToJSON(sseResponse: Response): Promise { + const reader = sseResponse.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + let eventType = ''; + let dataLines: string[] = []; + let resultData: unknown = null; + + const dispatch = () => { + const data = dataLines.join('\n'); + const type = eventType || 'message'; + eventType = ''; + dataLines = []; + if (type === 'result' && data) { + try { + resultData = JSON.parse(data); + } catch { + // keep previous resultData + } + } + }; + + const parseLine = (rawLine: string) => { + const line = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine; + if (line.startsWith(':')) return; + if (line === '') { dispatch(); return; } + const colon = line.indexOf(':'); + const field = colon === -1 ? line : line.slice(0, colon); + let val = colon === -1 ? '' : line.slice(colon + 1); + if (val.startsWith(' ')) val = val.slice(1); + if (field === 'event') eventType = val || 'message'; + else if (field === 'data') dataLines.push(val); + }; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) parseLine(line); + } + + // Flush remaining bytes and process trailing content + buffer += decoder.decode(); + if (buffer) { + for (const line of buffer.split('\n')) parseLine(line); + } + // Final dispatch for servers missing trailing blank line + if (dataLines.length > 0) dispatch(); + } catch { + // stream error, return what we have + } finally { + reader.releaseLock(); + } + + const body = resultData ?? { success: false, error: 'No result' }; + return new Response(JSON.stringify(body), { + headers: { 'Content-Type': 'application/json' } + }); +} + +/** + * Job-based response for long-running operations. + * + * Backward compat: API clients that send `Accept: application/json` (and not + * `text/event-stream`) get a synchronous JSON result directly. + * + * All other clients receive `{ jobId }` immediately. The operation runs in the + * background and results accumulate in the job store. Clients poll /api/jobs/{id}. + * + * The send() callback stores lines with { event, data } so the polling client + * can reconstruct the same event stream semantics used by the old SSE flow. + */ +export function createJobResponse( + operation: (send: (event: string, data: unknown) => void) => Promise, + request?: Request +): Response { + // Backward compat: synchronous JSON path for explicit application/json callers + if (prefersJSON(request)) { + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + async start(controller) { + let resultData: unknown = { success: false, error: 'No result' }; + const send = (_event: string, data: unknown) => { + resultData = data; + }; + try { + await operation(send); + } catch (error) { + resultData = { success: false, error: String(error) }; + } + controller.enqueue(encoder.encode(JSON.stringify(resultData))); + controller.close(); + } + }); + return new Response(stream, { + headers: { 'Content-Type': 'application/json' } + }); + } + + // Fire and forget: create job, run operation in background, return jobId immediately + const job = createJob(); + + const send = (event: string, data: unknown) => { + appendLine(job, { event, data }); + }; + + operation(send) + .then(() => { + const resultLine = job.lines.findLast((l) => l.event === 'result'); + completeJob(job, resultLine?.data ?? { success: true }); + }) + .catch((err: unknown) => { + failJob(job, err instanceof Error ? err.message : String(err)); + }); + + return json({ jobId: job.id }); +} diff --git a/src/lib/server/stack-scanner.ts b/src/lib/server/stack-scanner.ts index db0bad2..a344568 100644 --- a/src/lib/server/stack-scanner.ts +++ b/src/lib/server/stack-scanner.ts @@ -5,7 +5,7 @@ * Discovered stacks are editable - compose and .env files are modified in their original location. */ -import { readdirSync, existsSync, statSync } from 'node:fs'; +import { readdirSync, existsSync, statSync, readFileSync } from 'node:fs'; import { join, basename, dirname, resolve } from 'node:path'; import yaml from 'js-yaml'; import { getExternalStackPaths, getStackSources, upsertStackSource, type StackSourceType } from './db'; @@ -57,8 +57,7 @@ export function normalizeStackName(name: string): string { */ async function isComposeFile(filePath: string): Promise { try { - const file = Bun.file(filePath); - const content = await file.text(); + const content = readFileSync(filePath, 'utf-8'); // Basic check for services key - could be more sophisticated return /^services:/m.test(content) || /\nservices:/m.test(content); } catch { @@ -72,8 +71,7 @@ async function isComposeFile(filePath: string): Promise { */ async function countServices(filePath: string): Promise { try { - const file = Bun.file(filePath); - const content = await file.text(); + const content = readFileSync(filePath, 'utf-8'); const doc = yaml.load(content) as Record | null; if (doc?.services && typeof doc.services === 'object') { return Object.keys(doc.services).length; diff --git a/src/lib/server/stacks.ts b/src/lib/server/stacks.ts index 818fa74..e169084 100644 --- a/src/lib/server/stacks.ts +++ b/src/lib/server/stacks.ts @@ -7,6 +7,8 @@ import { existsSync, mkdirSync, rmSync, readdirSync, cpSync, statSync, unlinkSync, renameSync, readFileSync, writeFileSync } from 'node:fs'; import { join, resolve, dirname, basename } from 'node:path'; +import { spawn as nodeSpawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; import { getEnvironment, getSecretEnvVarsAsRecord, @@ -178,13 +180,47 @@ async function withStackLock(stackName: string, fn: () => Promise): Promis } } -// Timeout configuration for compose operations -const COMPOSE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes +// Timeout configuration for compose operations (configurable via COMPOSE_TIMEOUT env var in seconds) +const COMPOSE_TIMEOUT_MS = parseInt(process.env.COMPOSE_TIMEOUT || '900') * 1000; // Default 15 min const COMPOSE_KILL_GRACE_MS = 5000; // 5 seconds grace period before SIGKILL +/** + * Check if content is binary (not valid UTF-8 text). + */ +const utf8Decoder = new TextDecoder('utf-8', { fatal: true }); +function isBinaryContent(bytes: Uint8Array): boolean { + try { + utf8Decoder.decode(bytes); + return false; + } catch { + return true; + } +} + +/** + * Collect stdout/stderr from a child process and wait for it to exit. + */ +function collectProcess(proc: ChildProcess): Promise<{ exitCode: number; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + proc.stdout?.on('data', (chunk: Buffer) => stdoutChunks.push(chunk)); + proc.stderr?.on('data', (chunk: Buffer) => stderrChunks.push(chunk)); + proc.on('error', reject); + proc.on('close', (code) => { + resolve({ + exitCode: code ?? 1, + stdout: Buffer.concat(stdoutChunks).toString(), + stderr: Buffer.concat(stderrChunks).toString() + }); + }); + }); +} + /** * Read all files from a directory as a map of relative path -> content. * Used to send files to Hawser for remote deployments. + * Binary files are base64-encoded with a "base64:" prefix to preserve all bytes. */ async function readDirFilesAsMap(dirPath: string): Promise> { const files: Record = {}; @@ -200,9 +236,12 @@ async function readDirFilesAsMap(dirPath: string): Promise `${k}=${v}`); - await Bun.write(overrideEnvPath, header + lines.join('\n') + '\n'); + writeFileSync(overrideEnvPath, header + lines.join('\n') + '\n'); args.push('--env-file', overrideEnvPath); } @@ -1001,7 +1036,7 @@ async function executeLocalCompose( } break; case 'down': - args.push('down'); + args.push('down', '--remove-orphans'); if (removeVolumes) args.push('--volumes'); break; case 'stop': @@ -1045,12 +1080,10 @@ async function executeLocalCompose( try { console.log(`${logPrefix} Spawning docker compose process...`); - const proc = Bun.spawn(args, { + const proc = nodeSpawn(args[0], args.slice(1), { cwd: stackDir, env: spawnEnv, - stdin: useStdin ? 'pipe' : 'inherit', - stdout: 'pipe', - stderr: 'pipe' + stdio: [useStdin ? 'pipe' : 'inherit', 'pipe', 'pipe'] }); // If using stdin (host path translation), write the modified compose content @@ -1077,12 +1110,7 @@ async function executeLocalCompose( }, COMPOSE_TIMEOUT_MS); try { - const [stdout, stderr] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text() - ]); - - const code = await proc.exited; + const { exitCode: code, stdout, stderr } = await collectProcess(proc); console.log(`${logPrefix} ----------------------------------------`); console.log(`${logPrefix} COMPOSE PROCESS COMPLETE`); @@ -1343,7 +1371,7 @@ async function executeComposeCommand( let hawserEnvVars = envVars; if (envPath && existsSync(envPath)) { try { - const envFileContent = await Bun.file(envPath).text(); + const envFileContent = readFileSync(envPath, 'utf-8'); const envFileVars = parseEnvFileContent(envFileContent, stackName); // Merge: envFileVars (lowest) < envVars (DB overrides) // secretVars are handled separately in executeComposeViaHawser @@ -1362,7 +1390,7 @@ async function executeComposeCommand( const overridePath = findComposeOverrideFile(composeDir, composeBaseName); if (overridePath) { try { - const overrideContent = await Bun.file(overridePath).text(); + const overrideContent = readFileSync(overridePath, 'utf-8'); hawserStackFiles = { ...(hawserStackFiles || {}), [basename(overridePath)]: overrideContent }; console.log(`[Stack:${stackName}] Including override file for Hawser: ${basename(overridePath)}`); } catch (err) { @@ -1371,6 +1399,16 @@ async function executeComposeCommand( } } + // For git stacks: generate .env.dockhand with non-secret DB overrides + // This mirrors executeLocalCompose behavior (lines 1017-1023). + // envVars contains only the DB overrides (not merged repo .env values from hawserEnvVars). + if (useOverrideFile && envVars && Object.keys(envVars).length > 0) { + const header = '# Auto-generated by Dockhand. Do not edit - changes will be overwritten on next deploy.\n'; + const lines = Object.entries(envVars).map(([k, v]) => `${k}=${v}`); + hawserStackFiles = { ...(hawserStackFiles || {}), '.env.dockhand': header + lines.join('\n') + '\n' }; + console.log(`[Stack:${stackName}] Including .env.dockhand override file for Hawser (${Object.keys(envVars).length} vars)`); + } + return executeComposeViaHawser( operation, stackName, @@ -1766,8 +1804,9 @@ async function requireComposeFile( } /** - * Start a stack using docker compose up - * Falls back to individual container start for stacks without compose files + * Start a stack using docker compose start (resumes stopped containers). + * Falls back to docker compose up if containers don't exist (stack was removed/down). + * Falls back to individual container start for stacks without compose files. */ export async function startStack( stackName: string, @@ -1780,9 +1819,17 @@ export async function startStack( return withContainerFallback(stackName, envId, 'start'); } + const opts = { stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath }; + + // Check if containers exist for this stack. If they do, use 'start' to resume + // them (preserves container IDs, avoids Traefik race conditions from recreation). + // If no containers exist (stack was removed/down), use 'up' to create them. + const containers = await getStackContainers(stackName, envId); + const operation = containers.length > 0 ? 'start' : 'up'; + return executeComposeCommand( - 'up', - { stackName, envId, workingDir: result.stackDir, composePath: result.composePath, envPath: result.envPath }, + operation, + opts, result.content!, result.nonSecretVars, result.secretVars @@ -2185,7 +2232,7 @@ export async function deployStack(options: DeployStackOptions): Promise `${v.key.trim()}=${v.value}`) .join('\n') + '\n'; - await Bun.write(envFilePath, rawContent); + writeFileSync(envFilePath, rawContent); } /** @@ -2453,7 +2500,7 @@ export async function writeRawStackEnvFile( mkdirSync(dir, { recursive: true }); } - await Bun.write(envFilePath, rawContent); + writeFileSync(envFilePath, rawContent); } /** diff --git a/src/lib/server/subprocess-manager.ts b/src/lib/server/subprocess-manager.ts index fc1ac59..d14c4c1 100644 --- a/src/lib/server/subprocess-manager.ts +++ b/src/lib/server/subprocess-manager.ts @@ -1,633 +1,629 @@ /** * Subprocess Manager * - * Manages background subprocesses for metrics and event collection using Bun.spawn. - * Provides crash recovery, graceful shutdown, and IPC message routing. + * Manages a Go collection-worker process that handles background Docker API + * calls for metrics and event collection. Communication is via JSON lines + * over stdin (commands) / stdout (results). + * + * The Go worker handles: Docker API calls (ping, list, stats, info, df, events) + * This process handles: DB reads/writes, notifications, SSE broadcast */ -import { Subprocess } from 'bun'; -import { saveHostMetric, logContainerEvent, type ContainerEventAction } from './db'; -import { sendEventNotification, sendEnvironmentNotification } from './notifications'; -import { containerEventEmitter } from './event-collector'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { join } from 'node:path'; import { existsSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { containerEventEmitter } from './event-collector'; +import { + getEnvironments, + getEnvSetting, + getMetricsCollectionInterval, + getEventCollectionMode, + getEventPollInterval, + logContainerEvent, + type ContainerEventAction +} from './db'; +import { sendEnvironmentNotification, sendEventNotification } from './notifications'; +import { rssBeforeOp, rssAfterOp } from './rss-tracker'; +import { pushMetric } from './metrics-store'; -// Get the directory of this file (works in both Vite and Bun) -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- -// Determine subprocess script paths -// In development: src/lib/server/subprocesses/*.ts (via __dirname) -// In production: /app/subprocesses/*.js (bundled by scripts/build-subprocesses.ts) -function getSubprocessPath(name: string): string { - // Production path (Docker container) - bundled JS files - const prodPath = `/app/subprocesses/${name}.js`; - if (existsSync(prodPath)) { - return prodPath; - } - // Development path (relative to this file) - raw TS files - return path.join(__dirname, 'subprocesses', `${name}.ts`); -} - -// IPC Message Types (Subprocess → Main) -export interface MetricMessage { - type: 'metric'; - envId: number; - cpu: number; - memPercent: number; - memUsed: number; - memTotal: number; -} - -export interface DiskWarningMessage { - type: 'disk_warning'; - envId: number; - envName: string; - message: string; - diskPercent?: number; -} - -export interface ContainerEventMessage { - type: 'container_event'; - event: { - environmentId: number; - containerId: string; - containerName: string | null; - image: string | null; - action: ContainerEventAction; - actorAttributes: Record | null; - timestamp: string; - }; - notification?: { - action: ContainerEventAction; - title: string; - message: string; - notificationType: 'success' | 'error' | 'warning' | 'info'; - image?: string; - }; -} - -export interface EnvStatusMessage { - type: 'env_status'; - envId: number; - envName: string; - online: boolean; +interface GoMessage { + type: string; + envId?: number; + online?: boolean; error?: string; + event?: DockerEvent; + data?: any; + info?: any; // Docker /info response (for disk usage percentage) + cpu?: number; + memPercent?: number; + memUsed?: number; + memTotal?: number; + cpuCount?: number; } -export interface ReadyMessage { - type: 'ready'; +interface DockerEvent { + Type: string; + Action: string; + Actor: { ID: string; Attributes: Record }; + time: number; + timeNano: number; } -export interface ErrorMessage { - type: 'error'; - message: string; +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const CONTAINER_ACTIONS: ContainerEventAction[] = [ + 'create', 'start', 'stop', 'die', 'kill', 'restart', + 'pause', 'unpause', 'destroy', 'rename', 'update', 'oom', 'health_status' +]; + +const SCANNER_IMAGE_PATTERNS = [ + 'anchore/grype', 'aquasec/trivy', + 'ghcr.io/anchore/grype', 'ghcr.io/aquasecurity/trivy' +]; + +const EXCLUDED_CONTAINER_PREFIXES = ['dockhand-browse-']; + +const DEDUP_WINDOW_MS = 5000; +const MAX_DEDUP_CACHE_SIZE = 500; +const DISK_WARNING_COOLDOWN = 3600000; + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +let proc: ChildProcess | null = null; +let isShuttingDown = false; +let lineBuffer: Buffer = Buffer.alloc(0); +let restartDelay = 1000; +const MAX_RESTART_DELAY = 60000; + +// Dedup cache for events +const recentEvents: Map = new Map(); +// Disk warning cooldown per env +const lastDiskWarning: Map = new Map(); +// Environment name cache (for notifications) +const envNames: Map = new Map(); +// Track which envIds are currently configured in Go +const configuredEnvs: Set = new Set(); + +// Dedup cleanup interval +let dedupCleanupInterval: ReturnType | null = null; + +// --------------------------------------------------------------------------- +// Go binary path resolution +// --------------------------------------------------------------------------- + +function resolveWorkerPath(): string { + // Dev: pre-built binary in bin/ + const devPath = join(process.cwd(), 'bin', 'collection-worker'); + if (existsSync(devPath)) return devPath; + + // Production: alongside the app + const prodPath = join(process.cwd(), 'collection-worker'); + if (existsSync(prodPath)) return prodPath; + + // Docker: /app/bin/collection-worker + const dockerPath = '/app/bin/collection-worker'; + if (existsSync(dockerPath)) return dockerPath; + + throw new Error(`Go collection-worker not found at ${devPath}, ${prodPath}, or ${dockerPath}`); } -export type SubprocessMessage = - | MetricMessage - | DiskWarningMessage - | ContainerEventMessage - | EnvStatusMessage - | ReadyMessage - | ErrorMessage; +// --------------------------------------------------------------------------- +// IPC: send JSON line to Go process stdin +// --------------------------------------------------------------------------- -// IPC Message Types (Main → Subprocess) -export interface RefreshEnvironmentsCommand { - type: 'refresh_environments'; +function sendToGo(msg: Record): void { + if (!proc?.stdin || !proc.stdin.writable) return; + const line = JSON.stringify(msg) + '\n'; + proc.stdin.write(line); } -export interface ShutdownCommand { - type: 'shutdown'; -} +// --------------------------------------------------------------------------- +// IPC: handle JSON line from Go process stdout +// --------------------------------------------------------------------------- -export interface UpdateIntervalCommand { - type: 'update_interval'; - intervalMs: number; -} +function handleLine(line: string): void { + if (!line.trim()) return; -export type MainProcessCommand = RefreshEnvironmentsCommand | ShutdownCommand | UpdateIntervalCommand; - -// Subprocess configuration -interface SubprocessConfig { - name: string; - scriptPath: string; - restartDelayMs: number; - maxRestarts: number; -} - -// Subprocess state -interface SubprocessState { - process: Subprocess<'ignore', 'inherit', 'inherit'> | null; - restartCount: number; - lastRestartTime: number; - isShuttingDown: boolean; -} - -class SubprocessManager { - private metricsState: SubprocessState = { - process: null, - restartCount: 0, - lastRestartTime: 0, - isShuttingDown: false - }; - - private eventsState: SubprocessState = { - process: null, - restartCount: 0, - lastRestartTime: 0, - isShuttingDown: false - }; - - private readonly metricsConfig: SubprocessConfig = { - name: 'metrics-subprocess', - scriptPath: getSubprocessPath('metrics-subprocess'), - restartDelayMs: 5000, - maxRestarts: 10 - }; - - private readonly eventsConfig: SubprocessConfig = { - name: 'event-subprocess', - scriptPath: getSubprocessPath('event-subprocess'), - restartDelayMs: 5000, - maxRestarts: 10 - }; - - /** - * Start all subprocesses - */ - async start(): Promise { - console.log('[SubprocessManager] Starting background subprocesses...'); - - await this.startMetricsSubprocess(); - await this.startEventsSubprocess(); - - console.log('[SubprocessManager] All subprocesses started'); + const parseBefore = rssBeforeOp(); + let msg: GoMessage; + try { + msg = JSON.parse(line); + } catch { + console.error('[SubprocessManager] Invalid JSON from Go worker:', line.substring(0, 200)); + return; } + rssAfterOp('ipc_parse', parseBefore); - /** - * Stop all subprocesses gracefully - */ - async stop(): Promise { - console.log('[SubprocessManager] Stopping background subprocesses...'); + switch (msg.type) { + case 'ready': + console.log('[SubprocessManager] Go worker ready'); + restartDelay = 1000; // Reset backoff on successful start + break; - this.metricsState.isShuttingDown = true; - this.eventsState.isShuttingDown = true; + case 'metrics': + handleMetrics(msg); + break; - // Send shutdown commands - this.sendToMetrics({ type: 'shutdown' }); - this.sendToEvents({ type: 'shutdown' }); + case 'env_status': + handleEnvStatus(msg); + break; - // Wait a bit for graceful shutdown - await new Promise((resolve) => setTimeout(resolve, 1000)); + case 'container_event': + handleContainerEvent(msg); + break; - // Force kill if still running - if (this.metricsState.process) { - this.metricsState.process.kill(); - this.metricsState.process = null; - } - if (this.eventsState.process) { - this.eventsState.process.kill(); - this.eventsState.process = null; - } + case 'disk_usage': + handleDiskUsage(msg); + break; - console.log('[SubprocessManager] All subprocesses stopped'); - } - - /** - * Notify subprocesses to refresh their environment list - */ - refreshEnvironments(): void { - this.sendToMetrics({ type: 'refresh_environments' }); - this.sendToEvents({ type: 'refresh_environments' }); - } - - /** - * Send message to metrics subprocess - */ - sendToMetricsSubprocess(message: MainProcessCommand): void { - this.sendToMetrics(message); - } - - /** - * Send message to events subprocess - */ - sendToEventsSubprocess(message: MainProcessCommand): void { - this.sendToEvents(message); - } - - /** - * Start the metrics collection subprocess - */ - private async startMetricsSubprocess(): Promise { - if (this.metricsState.isShuttingDown) return; - - try { - console.log(`[SubprocessManager] Starting ${this.metricsConfig.name}...`); - - const proc = Bun.spawn(['bun', 'run', this.metricsConfig.scriptPath], { - stdio: ['inherit', 'inherit', 'inherit'], - env: { ...process.env, SKIP_MIGRATIONS: '1' }, - ipc: (message) => this.handleMetricsMessage(message as SubprocessMessage), - onExit: (proc, exitCode, signalCode) => { - this.handleMetricsExit(exitCode, signalCode); - } - }); - - this.metricsState.process = proc; - this.metricsState.restartCount = 0; - - console.log(`[SubprocessManager] ${this.metricsConfig.name} started (PID: ${proc.pid})`); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error(`[SubprocessManager] Failed to start ${this.metricsConfig.name}: ${msg}`); - this.scheduleMetricsRestart(); - } - } - - /** - * Start the event collection subprocess - */ - private async startEventsSubprocess(): Promise { - if (this.eventsState.isShuttingDown) return; - - try { - console.log(`[SubprocessManager] Starting ${this.eventsConfig.name}...`); - - const proc = Bun.spawn(['bun', 'run', this.eventsConfig.scriptPath], { - stdio: ['inherit', 'inherit', 'inherit'], - env: { ...process.env, SKIP_MIGRATIONS: '1' }, - ipc: (message) => this.handleEventsMessage(message as SubprocessMessage), - onExit: (proc, exitCode, signalCode) => { - this.handleEventsExit(exitCode, signalCode); - } - }); - - this.eventsState.process = proc; - this.eventsState.restartCount = 0; - - console.log(`[SubprocessManager] ${this.eventsConfig.name} started (PID: ${proc.pid})`); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error(`[SubprocessManager] Failed to start ${this.eventsConfig.name}: ${msg}`); - this.scheduleEventsRestart(); - } - } - - /** - * Handle IPC messages from metrics subprocess - */ - private async handleMetricsMessage(message: SubprocessMessage): Promise { - try { - switch (message.type) { - case 'ready': - console.log(`[SubprocessManager] ${this.metricsConfig.name} is ready`); - break; - - case 'metric': - // Save metric to database - await saveHostMetric( - message.cpu, - message.memPercent, - message.memUsed, - message.memTotal, - message.envId - ); - break; - - case 'disk_warning': - // Send disk warning notification - await sendEventNotification( - 'disk_space_warning', - { - title: message.diskPercent ? 'Disk space warning' : 'High Docker disk usage', - message: message.message, - type: 'warning' - }, - message.envId - ); - break; - - case 'error': - console.error(`[SubprocessManager] ${this.metricsConfig.name} error:`, message.message); - break; + case 'error': + if (msg.envId) { + console.warn(`[SubprocessManager] Go worker error for env ${msg.envId}: ${msg.error}`); + } else { + console.error(`[SubprocessManager] Go worker error: ${msg.error}`); } - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error(`[SubprocessManager] Error handling metrics message: ${msg}`); - } + break; + } +} + +// --------------------------------------------------------------------------- +// Message handlers +// --------------------------------------------------------------------------- + +function handleMetrics(msg: GoMessage): void { + if (!msg.envId || msg.cpu === undefined || msg.memPercent === undefined) return; + if (!configuredEnvs.has(msg.envId)) return; + + const before = rssBeforeOp(); + pushMetric(msg.envId, msg.cpu, msg.memPercent, msg.memUsed || 0, msg.memTotal || 0); + rssAfterOp('metrics', before); +} + +function handleEnvStatus(msg: GoMessage): void { + if (!msg.envId || msg.online === undefined) return; + + const before = rssBeforeOp(); + const envName = envNames.get(msg.envId) || `env-${msg.envId}`; + + containerEventEmitter.emit('env_status', { + envId: msg.envId, + envName, + online: msg.online, + error: msg.error + }); + + // Log status changes + if (msg.online) { + console.log(`[SubprocessManager] Environment "${envName}" (${msg.envId}) is now online`); + } else { + console.warn(`[SubprocessManager] Environment "${envName}" (${msg.envId}) is offline${msg.error ? `: ${msg.error}` : ''}`); } - /** - * Handle IPC messages from events subprocess - */ - private async handleEventsMessage(message: SubprocessMessage): Promise { - try { - switch (message.type) { - case 'ready': - console.log(`[SubprocessManager] ${this.eventsConfig.name} is ready`); - break; + // Send notifications for status changes + if (msg.online) { + sendEventNotification('environment_online', { + title: 'Environment online', + message: `Environment "${envName}" is now reachable`, + type: 'success' + }, msg.envId).catch((err) => { + console.error('[SubprocessManager] Failed to send online notification:', err instanceof Error ? err.message : String(err)); + }); + } else { + sendEventNotification('environment_offline', { + title: 'Environment offline', + message: `Environment "${envName}" is unreachable${msg.error ? `: ${msg.error}` : ''}`, + type: 'error' + }, msg.envId).catch((err) => { + console.error('[SubprocessManager] Failed to send offline notification:', err instanceof Error ? err.message : String(err)); + }); + } + rssAfterOp('status', before); +} - case 'container_event': - // Save event to database - const savedEvent = await logContainerEvent(message.event); +async function handleContainerEvent(msg: GoMessage): Promise { + if (!msg.envId || !msg.event) return; + if (!configuredEnvs.has(msg.envId)) return; - // Broadcast to SSE clients - containerEventEmitter.emit('event', savedEvent); + const before = rssBeforeOp(); + const event = msg.event; + if (event.Type !== 'container') return; - // Send notification if provided - if (message.notification) { - const { action, title, message: notifMessage, notificationType, image } = message.notification; - sendEnvironmentNotification(message.event.environmentId, action, { - title, - message: notifMessage, - type: notificationType - }, image).catch((err) => { - const errorMsg = err instanceof Error ? err.message : String(err); - console.error('[SubprocessManager] Failed to send notification:', errorMsg); - }); + const rawAction = event.Action; + const baseAction = rawAction.split(':')[0] as ContainerEventAction; + if (!CONTAINER_ACTIONS.includes(baseAction)) return; + + const action = rawAction.startsWith('health_status') ? rawAction : baseAction; + const containerId = event.Actor?.ID; + const containerName = event.Actor?.Attributes?.name; + const image = event.Actor?.Attributes?.image; + + if (!containerId) return; + if (image && SCANNER_IMAGE_PATTERNS.some(p => image.toLowerCase().includes(p.toLowerCase()))) return; + if (containerName && EXCLUDED_CONTAINER_PREFIXES.some(prefix => containerName.startsWith(prefix))) return; + + // Dedup + const dedupKey = `${msg.envId}-${event.timeNano}-${containerId}-${action}`; + if (recentEvents.has(dedupKey)) return; + recentEvents.set(dedupKey, Date.now()); + if (recentEvents.size > MAX_DEDUP_CACHE_SIZE) cleanupRecentEvents(); + + const timestamp = new Date(Math.floor(event.timeNano / 1000000)).toISOString(); + + // Sub-category: DB insert + const dbBefore = rssBeforeOp(); + try { + const savedEvent = await logContainerEvent({ + environmentId: msg.envId, + containerId, + containerName: containerName || null, + image: image || null, + action: action as ContainerEventAction, + actorAttributes: event.Actor?.Attributes || null, + timestamp + }); + + containerEventEmitter.emit('event', savedEvent); + } catch (err) { + console.error('[SubprocessManager] Failed to save event:', err instanceof Error ? err.message : String(err)); + } + rssAfterOp('events_db', dbBefore); + + // Sub-category: notification + const notifBefore = rssBeforeOp(); + const actionLabel = action.startsWith('health_status') + ? action.includes('unhealthy') ? 'Unhealthy' : 'Healthy' + : action.charAt(0).toUpperCase() + action.slice(1); + const containerLabel = containerName || containerId.substring(0, 12); + const notificationType = + action === 'die' || action === 'kill' || action === 'oom' || action.includes('unhealthy') + ? 'error' + : action === 'stop' + ? 'warning' + : action === 'start' || (action.includes('healthy') && !action.includes('unhealthy')) + ? 'success' + : 'info'; + + sendEnvironmentNotification(msg.envId, action, { + title: `Container ${actionLabel}`, + message: `Container "${containerLabel}" ${action}${image ? ` (${image})` : ''}`, + type: notificationType + }, image).catch(() => {}); + rssAfterOp('events_notif', notifBefore); + rssAfterOp('events', before); +} + +async function handleDiskUsage(msg: GoMessage): Promise { + if (!msg.envId || !msg.data) return; + if (!configuredEnvs.has(msg.envId)) return; + + const before = rssBeforeOp(); + const envName = envNames.get(msg.envId) || `env-${msg.envId}`; + + try { + const diskWarningEnabled = (await getEnvSetting('disk_warning_enabled', msg.envId)) ?? true; + if (!diskWarningEnabled) return; + + const lastWarning = lastDiskWarning.get(msg.envId); + if (lastWarning && Date.now() - lastWarning < DISK_WARNING_COOLDOWN) return; + + const diskData = msg.data; + let totalUsed = 0; + if (diskData.Images) totalUsed += diskData.Images.reduce((sum: number, img: any) => sum + (img.Size || 0), 0); + if (diskData.Containers) totalUsed += diskData.Containers.reduce((sum: number, c: any) => sum + (c.SizeRw || 0), 0); + if (diskData.Volumes) totalUsed += diskData.Volumes.reduce((sum: number, v: any) => sum + (v.UsageData?.Size || 0), 0); + if (diskData.BuildCache) totalUsed += diskData.BuildCache.reduce((sum: number, bc: any) => sum + (bc.Size || 0), 0); + + const diskWarningMode = (await getEnvSetting('disk_warning_mode', msg.envId)) ?? 'percentage'; + const GB = 1024 * 1024 * 1024; + + if (diskWarningMode === 'absolute') { + const thresholdGb = (await getEnvSetting('disk_warning_threshold_gb', msg.envId)) ?? 50; + if (totalUsed > thresholdGb * GB) { + await sendEventNotification('disk_space_warning', { + title: 'High Docker disk usage', + message: `Environment "${envName}" is using ${formatSize(totalUsed)} of Docker disk space (threshold: ${thresholdGb} GB)`, + type: 'warning' + }, msg.envId); + lastDiskWarning.set(msg.envId, Date.now()); + } + } else { + // Percentage mode — need DataSpaceTotal from /info DriverStatus + const driverStatus = msg.info?.DriverStatus; + let dataSpaceTotal = 0; + if (Array.isArray(driverStatus)) { + for (const [key, value] of driverStatus) { + if (key === 'Data Space Total' && typeof value === 'string') { + dataSpaceTotal = parseSize(value); + break; } - break; - - case 'env_status': - // Broadcast to dashboard via containerEventEmitter - containerEventEmitter.emit('env_status', { - envId: message.envId, - envName: message.envName, - online: message.online, - error: message.error - }); - - // Send environment status notification - if (message.online) { - await sendEventNotification( - 'environment_online', - { - title: 'Environment online', - message: `Environment "${message.envName}" is now reachable`, - type: 'success' - }, - message.envId - ).catch((err) => { - const errorMsg = err instanceof Error ? err.message : String(err); - console.error('[SubprocessManager] Failed to send online notification:', errorMsg); - }); - } else { - await sendEventNotification( - 'environment_offline', - { - title: 'Environment offline', - message: `Environment "${message.envName}" is unreachable${message.error ? `: ${message.error}` : ''}`, - type: 'error' - }, - message.envId - ).catch((err) => { - const errorMsg = err instanceof Error ? err.message : String(err); - console.error('[SubprocessManager] Failed to send offline notification:', errorMsg); - }); - } - break; - - case 'error': - console.error(`[SubprocessManager] ${this.eventsConfig.name} error:`, message.message); - break; + } } - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error(`[SubprocessManager] Error handling events message: ${msg}`); - } - } + if (dataSpaceTotal <= 0) return; - /** - * Handle metrics subprocess exit - */ - private handleMetricsExit(exitCode: number | null, signalCode: string | null): void { - if (this.metricsState.isShuttingDown) { - console.log(`[SubprocessManager] ${this.metricsConfig.name} stopped`); - return; - } - - console.error( - `[SubprocessManager] ${this.metricsConfig.name} exited unexpectedly (code: ${exitCode}, signal: ${signalCode})` - ); - - this.metricsState.process = null; - this.scheduleMetricsRestart(); - } - - /** - * Handle events subprocess exit - */ - private handleEventsExit(exitCode: number | null, signalCode: string | null): void { - if (this.eventsState.isShuttingDown) { - console.log(`[SubprocessManager] ${this.eventsConfig.name} stopped`); - return; - } - - console.error( - `[SubprocessManager] ${this.eventsConfig.name} exited unexpectedly (code: ${exitCode}, signal: ${signalCode})` - ); - - this.eventsState.process = null; - this.scheduleEventsRestart(); - } - - /** - * Schedule metrics subprocess restart with backoff - */ - private scheduleMetricsRestart(): void { - if (this.metricsState.isShuttingDown) return; - - if (this.metricsState.restartCount >= this.metricsConfig.maxRestarts) { - console.error( - `[SubprocessManager] ${this.metricsConfig.name} exceeded max restarts (${this.metricsConfig.maxRestarts}), giving up` - ); - return; - } - - const delay = this.metricsConfig.restartDelayMs * Math.pow(2, this.metricsState.restartCount); - this.metricsState.restartCount++; - - console.log( - `[SubprocessManager] Restarting ${this.metricsConfig.name} in ${delay}ms (attempt ${this.metricsState.restartCount}/${this.metricsConfig.maxRestarts})` - ); - - setTimeout(() => { - this.startMetricsSubprocess(); - }, delay); - } - - /** - * Schedule events subprocess restart with backoff - */ - private scheduleEventsRestart(): void { - if (this.eventsState.isShuttingDown) return; - - if (this.eventsState.restartCount >= this.eventsConfig.maxRestarts) { - console.error( - `[SubprocessManager] ${this.eventsConfig.name} exceeded max restarts (${this.eventsConfig.maxRestarts}), giving up` - ); - return; - } - - const delay = this.eventsConfig.restartDelayMs * Math.pow(2, this.eventsState.restartCount); - this.eventsState.restartCount++; - - console.log( - `[SubprocessManager] Restarting ${this.eventsConfig.name} in ${delay}ms (attempt ${this.eventsState.restartCount}/${this.eventsConfig.maxRestarts})` - ); - - setTimeout(() => { - this.startEventsSubprocess(); - }, delay); - } - - /** - * Send command to metrics subprocess - */ - private sendToMetrics(command: MainProcessCommand): void { - if (this.metricsState.process) { - try { - this.metricsState.process.send(command); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error(`[SubprocessManager] Failed to send to metrics subprocess: ${msg}`); + const diskPercentUsed = (totalUsed / dataSpaceTotal) * 100; + const threshold = (await getEnvSetting('disk_warning_threshold', msg.envId)) || 80; + if (diskPercentUsed >= threshold) { + console.log(`[SubprocessManager] Docker disk usage for ${envName}: ${diskPercentUsed.toFixed(1)}% (threshold: ${threshold}%)`); + await sendEventNotification('disk_space_warning', { + title: 'Disk space warning', + message: `Environment "${envName}" Docker disk usage is at ${diskPercentUsed.toFixed(1)}% (${formatSize(totalUsed)} used)`, + type: 'warning' + }, msg.envId); + lastDiskWarning.set(msg.envId, Date.now()); } } + } catch (err) { + console.error(`[SubprocessManager] Failed to process disk usage for env ${msg.envId}:`, err instanceof Error ? err.message : String(err)); + } + rssAfterOp('disk', before); +} + +function parseSize(sizeStr: string): number { + const units: Record = { + B: 1, KB: 1024, MB: 1024 ** 2, GB: 1024 ** 3, TB: 1024 ** 4 + }; + const match = sizeStr.match(/^([\d.]+)\s*([KMGT]?B)$/i); + if (!match) return 0; + return parseFloat(match[1]) * (units[match[2].toUpperCase()] || 1); +} + +function formatSize(bytes: number): string { + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let unitIndex = 0; + let size = bytes; + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex++; + } + return `${size.toFixed(1)} ${units[unitIndex]}`; +} + +function cleanupRecentEvents(): void { + const now = Date.now(); + for (const [key, timestamp] of recentEvents.entries()) { + if (now - timestamp > DEDUP_WINDOW_MS) { + recentEvents.delete(key); + } + } + if (recentEvents.size > MAX_DEDUP_CACHE_SIZE) { + const entries = Array.from(recentEvents.entries()).sort((a, b) => a[1] - b[1]); + const toRemove = entries.slice(0, entries.length - MAX_DEDUP_CACHE_SIZE); + for (const [key] of toRemove) recentEvents.delete(key); + } +} + +// --------------------------------------------------------------------------- +// Configure environments in Go worker +// --------------------------------------------------------------------------- + +async function sendEnvironmentConfigs(): Promise { + const environments = await getEnvironments(); + const activeIds = new Set(); + + for (const env of environments) { + // Skip hawser-edge (events come via WebSocket) + if (env.connectionType === 'hawser-edge') continue; + + activeIds.add(env.id); + envNames.set(env.id, env.name); + + // Build config matching Go's EnvConfig struct + let config: Record; + + if (env.connectionType === 'socket' || !env.connectionType) { + config = { + type: 'socket', + socketPath: env.socketPath || '/var/run/docker.sock' + }; + } else { + const protocol = (env.protocol as string) || 'http'; + config = { + type: protocol, + host: env.host || 'localhost', + port: env.port || 2375, + ca: env.tlsCa || undefined, + cert: env.tlsCert || undefined, + key: env.tlsKey || undefined, + skipVerify: !!env.tlsSkipVerify + }; + } + + // Only send if env has metrics or activity collection enabled + if (env.collectMetrics === false && env.collectActivity === false) continue; + + sendToGo({ + type: 'configure', + envId: env.id, + name: env.name, + config, + connectionType: env.connectionType || 'socket', + hawserToken: env.hawserToken || undefined + }); + + configuredEnvs.add(env.id); } - /** - * Send command to events subprocess - */ - private sendToEvents(command: MainProcessCommand): void { - if (this.eventsState.process) { - try { - this.eventsState.process.send(command); - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error(`[SubprocessManager] Failed to send to events subprocess: ${msg}`); - } + // Remove envs that are no longer active + for (const envId of configuredEnvs) { + if (!activeIds.has(envId)) { + sendToGo({ type: 'remove', envId }); + configuredEnvs.delete(envId); + envNames.delete(envId); } } - /** - * Get metrics subprocess PID (for HMR cleanup) - */ - getMetricsPid(): number | null { - return this.metricsState.process?.pid ?? null; - } + // Send settings + const metricsInterval = await getMetricsCollectionInterval(); + sendToGo({ type: 'set_metrics_interval', intervalMs: metricsInterval }); - /** - * Get events subprocess PID (for HMR cleanup) - */ - getEventsPid(): number | null { - return this.eventsState.process?.pid ?? null; - } + const eventMode = await getEventCollectionMode(); + const pollInterval = await getEventPollInterval(); + sendToGo({ type: 'set_event_mode', mode: eventMode, pollIntervalMs: pollInterval }); } -// Singleton instance -let manager: SubprocessManager | null = null; +// --------------------------------------------------------------------------- +// Process stdout reader (Node.js streams) +// --------------------------------------------------------------------------- -// Store PIDs globally to survive HMR reloads -// Using globalThis to persist across module reloads in dev mode -const GLOBAL_KEY = '__dockhand_subprocess_pids__'; -interface SubprocessPids { - metrics: number | null; - events: number | null; +function readStdout(): void { + if (!proc?.stdout) return; + + proc.stdout.on('data', (chunk: Buffer) => { + const readBefore = rssBeforeOp(); + + // Append chunk to buffer without string conversion + lineBuffer = lineBuffer.length === 0 ? chunk : Buffer.concat([lineBuffer, chunk]); + + // Extract complete lines (delimited by \n) + let start = 0; + for (let i = 0; i < lineBuffer.length; i++) { + if (lineBuffer[i] === 0x0a) { // newline + if (i > start) { + const line = lineBuffer.toString('utf8', start, i); + handleLine(line); + } + start = i + 1; + } + } + + // Keep leftover bytes (incomplete line). + // Buffer.from() copies the data to a new allocation, releasing the + // parent ArrayBuffer. Using subarray() would retain the entire chunk. + if (start === lineBuffer.length) { + lineBuffer = Buffer.alloc(0); + } else if (start > 0) { + lineBuffer = Buffer.from(lineBuffer.subarray(start)); + } + rssAfterOp('ipc_read', readBefore); + }); + + proc.stdout.on('error', (err) => { + if (!isShuttingDown) { + console.error('[SubprocessManager] stdout read error:', err.message); + } + }); } -function getStoredPids(): SubprocessPids { - return (globalThis as any)[GLOBAL_KEY] || { metrics: null, events: null }; -} - -function setStoredPids(pids: SubprocessPids): void { - (globalThis as any)[GLOBAL_KEY] = pids; -} +// --------------------------------------------------------------------------- +// Public API (unchanged interface) +// --------------------------------------------------------------------------- /** - * Kill any orphaned processes from previous HMR reloads - */ -function killOrphanedProcesses(): void { - const pids = getStoredPids(); - - if (pids.metrics) { - try { - process.kill(pids.metrics, 'SIGTERM'); - console.log(`[SubprocessManager] Killed orphaned metrics process (PID: ${pids.metrics})`); - } catch { - // Process already dead, ignore - } - } - - if (pids.events) { - try { - process.kill(pids.events, 'SIGTERM'); - console.log(`[SubprocessManager] Killed orphaned events process (PID: ${pids.events})`); - } catch { - // Process already dead, ignore - } - } - - setStoredPids({ metrics: null, events: null }); -} - -/** - * Start background subprocesses + * Start background Go collection worker. */ export async function startSubprocesses(): Promise { - // Kill any orphaned processes from HMR reloads - killOrphanedProcesses(); + if (isShuttingDown) return; - if (manager) { - console.warn('[SubprocessManager] Subprocesses already started'); + if (process.env.DISABLE_METRICS === 'true' && process.env.DISABLE_EVENTS === 'true') { + console.log('[SubprocessManager] Metrics and events both disabled, skipping worker'); return; } - manager = new SubprocessManager(); - await manager.start(); + const workerPath = resolveWorkerPath(); + console.log(`[SubprocessManager] Starting Go worker (${workerPath})...`); - // Store PIDs for HMR cleanup - setStoredPids({ - metrics: manager.getMetricsPid(), - events: manager.getEventsPid() + proc = spawn(workerPath, [], { + stdio: ['pipe', 'pipe', 'inherit'] + }); + + // Start reading stdout + readStdout(); + + // Handle process exit + proc.on('exit', (code) => { + if (!isShuttingDown) { + console.warn(`[SubprocessManager] Go worker exited with code ${code}, restarting in ${restartDelay / 1000}s...`); + proc = null; + configuredEnvs.clear(); + setTimeout(() => startSubprocesses(), restartDelay); + restartDelay = Math.min(restartDelay * 2, MAX_RESTART_DELAY); + } + }); + + proc.on('error', (err) => { + console.error('[SubprocessManager] Failed to start Go worker:', err.message); + proc = null; + }); + + // Wait a moment for the process to start, then send configs + await new Promise(resolve => setTimeout(resolve, 100)); + await sendEnvironmentConfigs(); + + // Start dedup cleanup interval + if (!dedupCleanupInterval) { + dedupCleanupInterval = setInterval(cleanupRecentEvents, 5000); + } +} + +/** + * Stop the background Go collection worker. + */ +export async function stopSubprocesses(): Promise { + isShuttingDown = true; + + if (dedupCleanupInterval) { + clearInterval(dedupCleanupInterval); + dedupCleanupInterval = null; + } + + if (proc) { + sendToGo({ type: 'shutdown' }); + + // Wait up to 2s for clean exit, then kill + await new Promise((resolve) => { + const timeout = setTimeout(() => { + if (proc) { + proc.kill(); + proc = null; + } + resolve(); + }, 2000); + + proc!.on('exit', () => { + clearTimeout(timeout); + proc = null; + resolve(); + }); + }); + } + + recentEvents.clear(); + lastDiskWarning.clear(); + configuredEnvs.clear(); +} + +/** + * Signal the worker to refresh its environment/event configuration. + */ +export function refreshSubprocessEnvironments(): void { + sendEnvironmentConfigs().catch(err => { + console.error('[SubprocessManager] Failed to refresh configs:', err instanceof Error ? err.message : String(err)); }); } /** - * Stop background subprocesses + * Send a command to the metrics worker (update_interval). */ -export async function stopSubprocesses(): Promise { - if (manager) { - await manager.stop(); - manager = null; - } - setStoredPids({ metrics: null, events: null }); -} - -/** - * Notify subprocesses to refresh environments - */ -export function refreshSubprocessEnvironments(): void { - if (manager) { - manager.refreshEnvironments(); +export function sendToMetricsSubprocess(message: { type: string; intervalMs?: number }): void { + if (message.type === 'update_interval' && message.intervalMs) { + sendToGo({ type: 'set_metrics_interval', intervalMs: message.intervalMs }); } } /** - * Send message to event subprocess + * Send a command to the event worker (refresh_environments). */ -export function sendToEventSubprocess(message: MainProcessCommand): void { - if (manager) { - manager.sendToEventsSubprocess(message); - } -} - -/** - * Send message to metrics subprocess - */ -export function sendToMetricsSubprocess(message: MainProcessCommand): void { - if (manager) { - manager.sendToMetricsSubprocess(message); +export function sendToEventSubprocess(message: { type: string }): void { + if (message.type === 'refresh_environments') { + refreshSubprocessEnvironments(); } } diff --git a/src/lib/server/subprocesses/event-subprocess.ts b/src/lib/server/subprocesses/event-subprocess.ts deleted file mode 100644 index d061341..0000000 --- a/src/lib/server/subprocesses/event-subprocess.ts +++ /dev/null @@ -1,687 +0,0 @@ -/** - * Event Collection Subprocess - * - * Runs as a separate process via Bun.spawn to collect Docker container events - * without blocking the main HTTP thread. - * - * Communication with main process via IPC (process.send). - */ - -import { getEnvironments, getEventCollectionMode, getEventPollInterval, type ContainerEventAction } from '../db'; -import { getDockerEvents } from '../docker'; -import type { MainProcessCommand } from '../subprocess-manager'; - -// Reconnection settings -const RECONNECT_DELAY = 5000; // 5 seconds -const MAX_RECONNECT_DELAY = 60000; // 1 minute max - -// Track environment online status for notifications -// Only send notifications on status CHANGES, not on every reconnect attempt -const environmentOnlineStatus: Map = new Map(); - -// Active collectors per environment (for streaming mode) -const collectors: Map | null }> = new Map(); - -// Poll intervals per environment (for polling mode) -const pollIntervals: Map> = new Map(); - -// Last poll timestamp per environment (for polling mode) -const lastPollTime: Map = new Map(); - -// Recent event cache for deduplication (key: timeNano-containerId-action) -const recentEvents: Map = new Map(); -const DEDUP_WINDOW_MS = 5000; // 5 second window for deduplication -const CACHE_CLEANUP_INTERVAL_MS = 5000; // Clean up cache every 5 seconds (match dedup window) -const MAX_DEDUP_CACHE_SIZE = 500; // Hard limit to prevent unbounded growth - -let cacheCleanupInterval: ReturnType | null = null; -let isShuttingDown = false; - -// Track current settings to detect changes -let currentPollInterval: number = 60000; -let currentMode: 'stream' | 'poll' = 'stream'; - -// Actions we care about for container activity -const CONTAINER_ACTIONS: ContainerEventAction[] = [ - 'create', - 'start', - 'stop', - 'die', - 'kill', - 'restart', - 'pause', - 'unpause', - 'destroy', - 'rename', - 'update', - 'oom', - 'health_status' -]; - -// Scanner image patterns to exclude from events -const SCANNER_IMAGE_PATTERNS = [ - 'anchore/grype', - 'aquasec/trivy', - 'ghcr.io/anchore/grype', - 'ghcr.io/aquasecurity/trivy' -]; - -// Container name patterns to exclude from events -const EXCLUDED_CONTAINER_PREFIXES = ['dockhand-browse-']; - -/** - * Send message to main process - */ -function send(message: any): void { - if (process.send) { - process.send(message); - } -} - -function isScannerContainer(image: string | null | undefined): boolean { - if (!image) return false; - const lowerImage = image.toLowerCase(); - return SCANNER_IMAGE_PATTERNS.some((pattern) => lowerImage.includes(pattern.toLowerCase())); -} - -function isExcludedContainer(containerName: string | null | undefined): boolean { - if (!containerName) return false; - return EXCLUDED_CONTAINER_PREFIXES.some((prefix) => containerName.startsWith(prefix)); -} - -/** - * Update environment online status and notify main process on change - */ -function updateEnvironmentStatus( - envId: number, - envName: string, - isOnline: boolean, - errorMessage?: string -) { - const previousStatus = environmentOnlineStatus.get(envId); - - // Only send notification on status CHANGE (not on first connection or repeated failures) - if (previousStatus !== undefined && previousStatus !== isOnline) { - send({ - type: 'env_status', - envId, - envName, - online: isOnline, - error: errorMessage - }); - } - - environmentOnlineStatus.set(envId, isOnline); -} - -interface DockerEvent { - Type: string; - Action: string; - Actor: { - ID: string; - Attributes: Record; - }; - time: number; - timeNano: number; -} - -/** - * Clean up old entries from the deduplication cache - * Also enforces max size limit with LRU eviction - */ -function cleanupRecentEvents() { - const now = Date.now(); - // First pass: remove expired entries - for (const [key, timestamp] of recentEvents.entries()) { - if (now - timestamp > DEDUP_WINDOW_MS) { - recentEvents.delete(key); - } - } - // Second pass: enforce max size with LRU eviction if still too large - if (recentEvents.size > MAX_DEDUP_CACHE_SIZE) { - const entries = Array.from(recentEvents.entries()) - .sort((a, b) => a[1] - b[1]); // Sort by timestamp (oldest first) - const toRemove = entries.slice(0, entries.length - MAX_DEDUP_CACHE_SIZE); - for (const [key] of toRemove) { - recentEvents.delete(key); - } - } -} - -/** - * Process a Docker event - */ -function processEvent(event: DockerEvent, envId: number) { - // Only process container events - if (event.Type !== 'container') return; - - // Map Docker action to our action type - // For health_status events, Docker sends "health_status: unhealthy" or "health_status: healthy" - // We need to preserve the full string for notifications to distinguish healthy vs unhealthy - const rawAction = event.Action; - const baseAction = rawAction.split(':')[0] as ContainerEventAction; - - // Skip actions we don't care about - if (!CONTAINER_ACTIONS.includes(baseAction)) return; - - // For notifications, preserve full action for health_status to enable proper mapping - const action = rawAction.startsWith('health_status') ? rawAction : baseAction; - - const containerId = event.Actor?.ID; - const containerName = event.Actor?.Attributes?.name; - const image = event.Actor?.Attributes?.image; - - if (!containerId) return; - - // Skip scanner containers (Trivy, Grype) - if (isScannerContainer(image)) return; - - // Skip internal Dockhand containers (volume browser helpers) - if (isExcludedContainer(containerName)) return; - - // Deduplicate events - const dedupKey = `${envId}-${event.timeNano}-${containerId}-${action}`; - if (recentEvents.has(dedupKey)) { - return; - } - - // Mark as processed - recentEvents.set(dedupKey, Date.now()); - - // Clean up if cache gets too large - if (recentEvents.size > 200) { - cleanupRecentEvents(); - } - - // Convert Unix nanosecond timestamp to ISO string - const timestamp = new Date(Math.floor(event.timeNano / 1000000)).toISOString(); - - // Prepare notification data - // For health_status events, create a cleaner label - const actionLabel = action.startsWith('health_status') - ? action.includes('unhealthy') ? 'Unhealthy' : 'Healthy' - : action.charAt(0).toUpperCase() + action.slice(1); - const containerLabel = containerName || containerId.substring(0, 12); - const notificationType = - action === 'die' || action === 'kill' || action === 'oom' || action.includes('unhealthy') - ? 'error' - : action === 'stop' - ? 'warning' - : action === 'start' || (action.includes('healthy') && !action.includes('unhealthy')) - ? 'success' - : 'info'; - - // Send event to main process for DB save and SSE broadcast - send({ - type: 'container_event', - event: { - environmentId: envId, - containerId: containerId, - containerName: containerName || null, - image: image || null, - action, - actorAttributes: event.Actor?.Attributes || null, - timestamp - }, - notification: { - action, - title: `Container ${actionLabel}`, - message: `Container "${containerLabel}" ${action}${image ? ` (${image})` : ''}`, - notificationType, - image - } - }); -} - -/** - * Poll events for a specific environment (polling mode) - */ -async function pollEnvironmentEvents(envId: number, envName: string) { - try { - // Calculate 'since' timestamp (use last poll time, or start from 30s ago if first poll) - const now = Math.floor(Date.now() / 1000); // Unix timestamp in seconds - const since = lastPollTime.get(envId) || (now - 30); // Default to 30s ago on first poll - - // Fetch events since last check until now - // IMPORTANT: 'until' is required for polling mode, otherwise Docker keeps the connection open - const eventStream = await getDockerEvents( - { type: ['container'] }, - envId, - { since: since.toString(), until: now.toString() } - ); - - if (!eventStream) { - console.error(`[EventSubprocess] Failed to fetch events for ${envName}`); - updateEnvironmentStatus(envId, envName, false, 'Failed to fetch Docker events'); - return; - } - - // Mark environment as online - updateEnvironmentStatus(envId, envName, true); - - // Read and process all events - const reader = eventStream.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - if (line.trim()) { - try { - const event = JSON.parse(line) as DockerEvent; - processEvent(event, envId); - } catch { - // Ignore parse errors - } - } - } - } - } finally { - try { - // Cancel the stream first to ensure proper cleanup, then release lock - await reader.cancel(); - reader.releaseLock(); - } catch { - // Reader already released or stream closed - } - } - - // Update last poll time - lastPollTime.set(envId, now); - - } catch (error: any) { - if (!isShuttingDown) { - console.error(`[EventSubprocess] Poll error for ${envName}:`, error.message); - updateEnvironmentStatus(envId, envName, false, error.message); - } - } -} - -/** - * Start collecting events for a specific environment - */ -async function startEnvironmentCollector(envId: number, envName: string) { - // Stop existing collector if any - stopEnvironmentCollector(envId); - - const controller = new AbortController(); - const entry = { controller, reconnectTimeout: null as ReturnType | null }; - collectors.set(envId, entry); - - let reconnectDelay = RECONNECT_DELAY; - - const connect = async () => { - if (controller.signal.aborted || isShuttingDown) return; - - let reader: ReadableStreamDefaultReader | null = null; - - try { - console.log( - `[EventSubprocess] Connecting to Docker events for ${envName} (env ${envId})...` - ); - - const eventStream = await getDockerEvents({ type: ['container'] }, envId); - - if (!eventStream) { - console.error(`[EventSubprocess] Failed to get event stream for ${envName}`); - updateEnvironmentStatus(envId, envName, false, 'Failed to connect to Docker'); - scheduleReconnect(); - return; - } - - // Reset reconnect delay on successful connection - reconnectDelay = RECONNECT_DELAY; - console.log(`[EventSubprocess] Connected to Docker events for ${envName}`); - - updateEnvironmentStatus(envId, envName, true); - - reader = eventStream.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - - try { - while (!controller.signal.aborted && !isShuttingDown) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split('\n'); - buffer = lines.pop() || ''; - - for (const line of lines) { - if (line.trim()) { - try { - const event = JSON.parse(line) as DockerEvent; - processEvent(event, envId); - } catch { - // Ignore parse errors for partial chunks - } - } - } - } - } catch (error: any) { - if (!controller.signal.aborted && !isShuttingDown) { - if (error.name !== 'AbortError') { - console.error(`[EventSubprocess] Stream error for ${envName}:`, error.message); - updateEnvironmentStatus(envId, envName, false, error.message); - } - } - } finally { - if (reader) { - try { - // Cancel the stream first to ensure proper cleanup, then release lock - await reader.cancel(); - reader.releaseLock(); - } catch { - // Reader already released or stream closed - ignore - } - } - } - - // Connection closed, reconnect - if (!controller.signal.aborted && !isShuttingDown) { - scheduleReconnect(); - } - } catch (error: any) { - if (reader) { - try { - // Cancel the stream first to ensure proper cleanup, then release lock - await reader.cancel(); - reader.releaseLock(); - } catch { - // Reader already released or stream closed - ignore - } - } - - if (!controller.signal.aborted && !isShuttingDown && error.name !== 'AbortError') { - console.error(`[EventSubprocess] Connection error for ${envName}:`, error.message); - updateEnvironmentStatus(envId, envName, false, error.message); - } - - if (!controller.signal.aborted && !isShuttingDown) { - scheduleReconnect(); - } - } - }; - - const scheduleReconnect = () => { - if (controller.signal.aborted || isShuttingDown) return; - - console.log(`[EventSubprocess] Reconnecting to ${envName} in ${reconnectDelay / 1000}s...`); - entry.reconnectTimeout = setTimeout(() => { - entry.reconnectTimeout = null; - if (!controller.signal.aborted && !isShuttingDown) { - connect(); - } - }, reconnectDelay); - - // Exponential backoff - reconnectDelay = Math.min(reconnectDelay * 2, MAX_RECONNECT_DELAY); - }; - - // Start the connection - connect(); -} - -/** - * Start polling mode for a specific environment - */ -async function startEnvironmentPoller(envId: number, envName: string, interval: number) { - // Stop existing poller if any - stopEnvironmentPoller(envId); - - console.log(`[EventSubprocess] Starting poller for ${envName} (every ${interval / 1000}s)`); - - // Initial poll immediately - await pollEnvironmentEvents(envId, envName); - - // Set up interval for subsequent polls - const intervalId = setInterval(async () => { - if (!isShuttingDown) { - await pollEnvironmentEvents(envId, envName); - } - }, interval); - - pollIntervals.set(envId, intervalId); -} - -/** - * Stop polling for a specific environment - */ -function stopEnvironmentPoller(envId: number) { - const intervalId = pollIntervals.get(envId); - if (intervalId) { - clearInterval(intervalId); - pollIntervals.delete(envId); - lastPollTime.delete(envId); - environmentOnlineStatus.delete(envId); - } -} - -/** - * Stop collecting events for a specific environment (streaming mode) - */ -function stopEnvironmentCollector(envId: number) { - const entry = collectors.get(envId); - if (entry) { - if (entry.reconnectTimeout !== null) { - clearTimeout(entry.reconnectTimeout); - } - entry.controller.abort(); - collectors.delete(envId); - environmentOnlineStatus.delete(envId); - } -} - -/** - * Refresh collectors when environments change - */ -async function refreshEventCollectors() { - if (isShuttingDown) return; - - try { - const environments = await getEnvironments(); - const mode = await getEventCollectionMode(); - const pollInterval = await getEventPollInterval(); - - // Detect if settings changed - const modeChanged = mode !== currentMode; - const intervalChanged = pollInterval !== currentPollInterval; - - if (modeChanged) { - console.log(`[EventSubprocess] Mode changed from ${currentMode} to ${mode}`); - currentMode = mode; - } - if (intervalChanged) { - console.log(`[EventSubprocess] Poll interval changed from ${currentPollInterval}ms to ${pollInterval}ms`); - currentPollInterval = pollInterval; - } - - // Filter: only collect for environments with activity enabled AND not Hawser Edge - const activeEnvIds = new Set( - environments - .filter((e) => e.collectActivity && e.connectionType !== 'hawser-edge') - .map((e) => e.id) - ); - - // Stop collectors for removed environments or those with collection disabled - for (const envId of collectors.keys()) { - if (!activeEnvIds.has(envId)) { - console.log(`[EventSubprocess] Stopping stream collector for environment ${envId}`); - stopEnvironmentCollector(envId); - } - } - - // Stop pollers for removed environments or those with collection disabled - // Also restart all pollers if interval changed - for (const envId of pollIntervals.keys()) { - if (!activeEnvIds.has(envId)) { - console.log(`[EventSubprocess] Stopping poller for environment ${envId}`); - stopEnvironmentPoller(envId); - } else if (intervalChanged && mode === 'poll') { - // Restart poller with new interval - console.log(`[EventSubprocess] Restarting poller for environment ${envId} with new interval`); - stopEnvironmentPoller(envId); - } - } - - // Clean up stale map entries for deleted environments - const allEnvIds = new Set(environments.map((e) => e.id)); - for (const envId of environmentOnlineStatus.keys()) { - if (!allEnvIds.has(envId)) environmentOnlineStatus.delete(envId); - } - for (const envId of lastPollTime.keys()) { - if (!allEnvIds.has(envId)) lastPollTime.delete(envId); - } - - // Start collectors based on mode - for (const env of environments) { - // Skip Hawser Edge (handled by main process) - if (env.connectionType === 'hawser-edge') continue; - - // Skip if activity collection is disabled - if (!env.collectActivity) continue; - - const hasStreamCollector = collectors.has(env.id); - const hasPoller = pollIntervals.has(env.id); - - if (mode === 'stream') { - // Switch from polling to streaming if needed - if (hasPoller) { - console.log(`[EventSubprocess] Switching ${env.name} from poll to stream`); - stopEnvironmentPoller(env.id); - } - // Start stream if not already running - if (!hasStreamCollector) { - startEnvironmentCollector(env.id, env.name); - } - } else if (mode === 'poll') { - // Switch from streaming to polling if needed - if (hasStreamCollector) { - console.log(`[EventSubprocess] Switching ${env.name} from stream to poll`); - stopEnvironmentCollector(env.id); - } - // Start poller if not already running (will also restart after interval change above) - if (!hasPoller) { - startEnvironmentPoller(env.id, env.name, pollInterval); - } - } - } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(`[EventSubprocess] Failed to refresh collectors: ${message}`); - send({ type: 'error', message: `Failed to refresh collectors: ${message}` }); - } -} - -/** - * Handle commands from main process - */ -function handleCommand(command: MainProcessCommand): void { - switch (command.type) { - case 'refresh_environments': - console.log('[EventSubprocess] Refreshing environments...'); - refreshEventCollectors(); - break; - - case 'update_interval': - // This is used by metrics subprocess, but we handle it here too for consistency - // Event subprocess re-reads interval from DB on refresh - console.log('[EventSubprocess] Interval update - refreshing collectors...'); - refreshEventCollectors(); - break; - - case 'shutdown': - console.log('[EventSubprocess] Shutdown requested'); - shutdown(); - break; - } -} - -/** - * Graceful shutdown - */ -function shutdown(): void { - isShuttingDown = true; - - // Stop periodic cache cleanup - if (cacheCleanupInterval) { - clearInterval(cacheCleanupInterval); - cacheCleanupInterval = null; - } - - // Stop all environment stream collectors - for (const envId of collectors.keys()) { - stopEnvironmentCollector(envId); - } - - // Stop all environment pollers - for (const envId of pollIntervals.keys()) { - stopEnvironmentPoller(envId); - } - - // Clear the deduplication cache - recentEvents.clear(); - - console.log('[EventSubprocess] Stopped'); - process.exit(0); -} - -/** - * Start the event collector - */ -async function start(): Promise { - console.log('[EventSubprocess] Starting container event collection...'); - - // Initialize current settings from database - try { - currentMode = await getEventCollectionMode(); - currentPollInterval = await getEventPollInterval(); - console.log(`[EventSubprocess] Initial mode: ${currentMode}, poll interval: ${currentPollInterval}ms`); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(`[EventSubprocess] Failed to load settings, using defaults: ${message}`); - } - - // Start collectors for all environments - await refreshEventCollectors(); - - // Start periodic cache cleanup - cacheCleanupInterval = setInterval(cleanupRecentEvents, CACHE_CLEANUP_INTERVAL_MS); - console.log('[EventSubprocess] Started deduplication cache cleanup (every 5s)'); - - // Start memory diagnostics logging (every 5 minutes) - setInterval(() => { - const mem = process.memoryUsage(); - console.log( - `[EventSubprocess] Memory: heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB, ` + - `rss=${Math.round(mem.rss / 1024 / 1024)}MB, ` + - `dedup=${recentEvents.size}, collectors=${collectors.size}, pollers=${pollIntervals.size}` - ); - }, 5 * 60 * 1000); - - // Listen for commands from main process - process.on('message', (message: MainProcessCommand) => { - handleCommand(message); - }); - - // Handle termination signals - process.on('SIGTERM', shutdown); - process.on('SIGINT', shutdown); - - // Signal ready - send({ type: 'ready' }); - - console.log('[EventSubprocess] Started successfully'); -} - -// Start the subprocess -start(); diff --git a/src/lib/server/subprocesses/metrics-subprocess.ts b/src/lib/server/subprocesses/metrics-subprocess.ts deleted file mode 100644 index 6b6dffd..0000000 --- a/src/lib/server/subprocesses/metrics-subprocess.ts +++ /dev/null @@ -1,498 +0,0 @@ -/** - * Metrics Collection Subprocess - * - * Runs as a separate process via Bun.spawn to collect CPU/memory metrics - * and check disk space without blocking the main HTTP thread. - * - * Communication with main process via IPC (process.send). - */ - -import { getEnvironments, getEnvSetting, getMetricsCollectionInterval } from '../db'; -import { listContainers, getContainerStats, getDockerInfo, getDiskUsage } from '../docker'; -import os from 'node:os'; -import type { MainProcessCommand } from '../subprocess-manager'; - -let COLLECT_INTERVAL = 30000; // 30 seconds (default, will be loaded from settings) -const DISK_CHECK_INTERVAL = 300000; // 5 minutes -const DEFAULT_DISK_THRESHOLD = 80; // 80% threshold for disk warnings -const ENV_METRICS_TIMEOUT = 15000; // 15 seconds timeout per environment for metrics -const ENV_DISK_TIMEOUT = 20000; // 20 seconds timeout per environment for disk checks - -/** - * Timeout wrapper - returns fallback if promise takes too long - * IMPORTANT: Properly clears the timeout to prevent memory leaks - */ -function withTimeout(promise: Promise, ms: number, fallback: T): Promise { - let timeoutId: ReturnType | null = null; - - const timeoutPromise = new Promise((resolve) => { - timeoutId = setTimeout(() => resolve(fallback), ms); - }); - - return Promise.race([promise, timeoutPromise]).finally(() => { - if (timeoutId !== null) { - clearTimeout(timeoutId); - } - }); -} - -// Track last disk warning sent per environment to avoid spamming -const lastDiskWarning: Map = new Map(); -const DISK_WARNING_COOLDOWN = 3600000; // 1 hour between warnings - -let collectInterval: ReturnType | null = null; -let diskCheckInterval: ReturnType | null = null; -let isShuttingDown = false; -let collectionCycleCount = 0; -const MEMORY_LOG_INTERVAL = 10; // Log memory every 10 cycles (~5 minutes at 30s interval) - -/** - * Send message to main process - */ -function send(message: any): void { - if (process.send) { - process.send(message); - } -} - -/** - * Collect metrics for a single environment - */ -async function collectEnvMetrics(env: { id: number; name: string; host?: string; socketPath?: string; collectMetrics?: boolean; connectionType?: string }) { - try { - // Skip environments where metrics collection is disabled - if (env.collectMetrics === false) { - return; - } - - // Skip Hawser Edge environments (handled by main process) - if (env.connectionType === 'hawser-edge') { - return; - } - - // Get running containers - const containers = await listContainers(false, env.id); // Only running - let totalCpuPercent = 0; - let totalContainerMemUsed = 0; - - // Get stats for each running container - const statsPromises = containers.map(async (container) => { - try { - const stats = (await getContainerStats(container.id, env.id)) as any; - - // Calculate CPU percentage - const cpuDelta = - stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage; - const systemDelta = - stats.cpu_stats.system_cpu_usage - stats.precpu_stats.system_cpu_usage; - const cpuCount = stats.cpu_stats.online_cpus || os.cpus().length; - - let cpuPercent = 0; - if (systemDelta > 0 && cpuDelta > 0) { - cpuPercent = (cpuDelta / systemDelta) * cpuCount * 100; - } - - // Get container memory usage using the same formula as Docker CLI - // Docker subtracts cache (inactive_file) from total usage - // - cgroup v2: uses 'inactive_file' - // - cgroup v1: uses 'total_inactive_file' - const memUsage = stats.memory_stats?.usage || 0; - const memStats = stats.memory_stats?.stats || {}; - const memCache = memStats.inactive_file ?? memStats.total_inactive_file ?? 0; - const actualMemUsed = memCache > 0 && memCache < memUsage ? memUsage - memCache : memUsage; - - return { cpuPercent, memUsage: actualMemUsed }; - } catch { - return { cpuPercent: 0, memUsage: 0 }; - } - }); - - const statsResults = await Promise.all(statsPromises); - totalCpuPercent = statsResults.reduce((sum, r) => sum + r.cpuPercent, 0); - totalContainerMemUsed = statsResults.reduce((sum, r) => sum + r.memUsage, 0); - - // Get host memory info from Docker - const info = (await getDockerInfo(env.id)) as any; - const memTotal = info?.MemTotal || os.totalmem(); - - // Calculate memory: sum of all container memory vs host total - const memUsed = totalContainerMemUsed; - const memPercent = memTotal > 0 ? (memUsed / memTotal) * 100 : 0; - - // Normalize CPU by number of cores from the Docker host - const cpuCount = info?.NCPU || os.cpus().length; - const normalizedCpu = totalCpuPercent / cpuCount; - - // Validate values - skip if any are NaN, Infinity, or negative - const finalCpu = Number.isFinite(normalizedCpu) && normalizedCpu >= 0 ? normalizedCpu : 0; - const finalMemPercent = Number.isFinite(memPercent) && memPercent >= 0 ? memPercent : 0; - const finalMemUsed = Number.isFinite(memUsed) && memUsed >= 0 ? memUsed : 0; - const finalMemTotal = Number.isFinite(memTotal) && memTotal > 0 ? memTotal : 0; - - // Only send if we have valid memory total (otherwise metrics are meaningless) - if (finalMemTotal > 0) { - send({ - type: 'metric', - envId: env.id, - cpu: finalCpu, - memPercent: finalMemPercent, - memUsed: finalMemUsed, - memTotal: finalMemTotal - }); - } - } catch (error) { - // Skip this environment if it fails (might be offline) - const message = error instanceof Error ? error.message : String(error); - console.warn(`[MetricsSubprocess] Failed to collect metrics for ${env.name}: ${message}`); - } -} - -/** - * Collect metrics for all environments - */ -async function collectMetrics() { - if (isShuttingDown) return; - - try { - const environments = await getEnvironments(); - - // Filter enabled environments and collect metrics in parallel - const enabledEnvs = environments.filter((env) => env.collectMetrics !== false); - - // Process all environments in parallel with per-environment timeouts - // Use Promise.allSettled so one slow/failed env doesn't block others - const results = await Promise.allSettled( - enabledEnvs.map((env) => - withTimeout( - collectEnvMetrics(env).then(() => env.name), - ENV_METRICS_TIMEOUT, - null - ) - ) - ); - - // Log any environments that timed out - results.forEach((result, index) => { - if (result.status === 'fulfilled' && result.value === null) { - console.warn(`[MetricsSubprocess] Environment "${enabledEnvs[index].name}" metrics timed out after ${ENV_METRICS_TIMEOUT}ms`); - } else if (result.status === 'rejected') { - const reason = result.reason instanceof Error ? result.reason.message : String(result.reason); - console.warn(`[MetricsSubprocess] Environment "${enabledEnvs[index].name}" metrics failed: ${reason}`); - } - }); - - // Periodic memory logging for diagnostics - collectionCycleCount++; - if (collectionCycleCount % MEMORY_LOG_INTERVAL === 0) { - const memUsage = process.memoryUsage(); - const heapMB = Math.round(memUsage.heapUsed / 1024 / 1024); - const rssMB = Math.round(memUsage.rss / 1024 / 1024); - console.log(`[MetricsSubprocess] Memory: heap=${heapMB}MB, rss=${rssMB}MB (cycle ${collectionCycleCount})`); - } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(`[MetricsSubprocess] Metrics collection error: ${message}`); - send({ type: 'error', message: `Metrics collection error: ${message}` }); - } -} - -/** - * Parse size string like "107.4GB" to bytes - */ -function parseSize(sizeStr: string): number { - const units: Record = { - B: 1, - KB: 1024, - MB: 1024 * 1024, - GB: 1024 * 1024 * 1024, - TB: 1024 * 1024 * 1024 * 1024 - }; - - const match = sizeStr.match(/^([\d.]+)\s*([KMGT]?B)$/i); - if (!match) return 0; - - const value = parseFloat(match[1]); - const unit = match[2].toUpperCase(); - return value * (units[unit] || 1); -} - -/** - * Format bytes to human readable string - */ -function formatSize(bytes: number): string { - const units = ['B', 'KB', 'MB', 'GB', 'TB']; - let unitIndex = 0; - let size = bytes; - - while (size >= 1024 && unitIndex < units.length - 1) { - size /= 1024; - unitIndex++; - } - - return `${size.toFixed(1)} ${units[unitIndex]}`; -} - -/** - * Check disk space for a single environment - */ -async function checkEnvDiskSpace(env: { id: number; name: string; collectMetrics?: boolean; connectionType?: string }) { - try { - // Skip environments where metrics collection is disabled - if (env.collectMetrics === false) { - return; - } - - // Skip Hawser Edge environments (handled by main process) - if (env.connectionType === 'hawser-edge') { - return; - } - - // Check if disk warnings are enabled for this environment - const diskWarningEnabled = (await getEnvSetting('disk_warning_enabled', env.id)) ?? true; - if (!diskWarningEnabled) { - return; - } - - // Check if we're in cooldown for this environment - const lastWarningTime = lastDiskWarning.get(env.id); - if (lastWarningTime && Date.now() - lastWarningTime < DISK_WARNING_COOLDOWN) { - return; // Skip this environment, still in cooldown - } - - // Get Docker disk usage data - const diskData = (await getDiskUsage(env.id)) as any; - if (!diskData) return; - - // Calculate total Docker disk usage using reduce for cleaner code - let totalUsed = 0; - if (diskData.Images) { - totalUsed += diskData.Images.reduce((sum: number, img: any) => sum + (img.Size || 0), 0); - } - if (diskData.Containers) { - totalUsed += diskData.Containers.reduce((sum: number, c: any) => sum + (c.SizeRw || 0), 0); - } - if (diskData.Volumes) { - totalUsed += diskData.Volumes.reduce( - (sum: number, v: any) => sum + (v.UsageData?.Size || 0), - 0 - ); - } - if (diskData.BuildCache) { - totalUsed += diskData.BuildCache.reduce((sum: number, bc: any) => sum + (bc.Size || 0), 0); - } - - // Get Docker root filesystem info from Docker info - const info = (await getDockerInfo(env.id)) as any; - const driverStatus = info?.DriverStatus; - - // Try to find "Data Space Total" from driver status - let dataSpaceTotal = 0; - let diskPercentUsed = 0; - - if (driverStatus) { - for (const [key, value] of driverStatus) { - if (key === 'Data Space Total' && typeof value === 'string') { - dataSpaceTotal = parseSize(value); - break; - } - } - } - - // Determine warning mode - const diskWarningMode = (await getEnvSetting('disk_warning_mode', env.id)) ?? 'percentage'; - const GB = 1024 * 1024 * 1024; - - if (diskWarningMode === 'absolute') { - // Absolute mode: warn when usage exceeds GB threshold - const thresholdGb = (await getEnvSetting('disk_warning_threshold_gb', env.id)) ?? 50; - if (totalUsed > thresholdGb * GB) { - send({ - type: 'disk_warning', - envId: env.id, - envName: env.name, - message: `Environment "${env.name}" is using ${formatSize(totalUsed)} of Docker disk space (threshold: ${thresholdGb} GB)` - }); - lastDiskWarning.set(env.id, Date.now()); - } - } else { - // Percentage mode: need total disk space - if (dataSpaceTotal > 0) { - diskPercentUsed = (totalUsed / dataSpaceTotal) * 100; - } else { - // Can't determine percentage without total space — skip - return; - } - - const threshold = - (await getEnvSetting('disk_warning_threshold', env.id)) || DEFAULT_DISK_THRESHOLD; - if (diskPercentUsed >= threshold) { - console.log( - `[MetricsSubprocess] Docker disk usage for ${env.name}: ${diskPercentUsed.toFixed(1)}% (threshold: ${threshold}%)` - ); - - send({ - type: 'disk_warning', - envId: env.id, - envName: env.name, - message: `Environment "${env.name}" Docker disk usage is at ${diskPercentUsed.toFixed(1)}% (${formatSize(totalUsed)} used)`, - diskPercent: diskPercentUsed - }); - - lastDiskWarning.set(env.id, Date.now()); - } - } - } catch (error) { - // Skip this environment if it fails - const message = error instanceof Error ? error.message : String(error); - console.warn(`[MetricsSubprocess] Failed to check disk space for ${env.name}: ${message}`); - } -} - -/** - * Check disk space for all environments - */ -async function checkDiskSpace() { - if (isShuttingDown) return; - - try { - const environments = await getEnvironments(); - - // Filter enabled environments and check disk space in parallel - const enabledEnvs = environments.filter((env) => env.collectMetrics !== false); - - // Process all environments in parallel with per-environment timeouts - // Use Promise.allSettled so one slow/failed env doesn't block others - const results = await Promise.allSettled( - enabledEnvs.map((env) => - withTimeout( - checkEnvDiskSpace(env).then(() => env.name), - ENV_DISK_TIMEOUT, - null - ) - ) - ); - - // Log any environments that timed out - results.forEach((result, index) => { - if (result.status === 'fulfilled' && result.value === null) { - console.warn(`[MetricsSubprocess] Environment "${enabledEnvs[index].name}" disk check timed out after ${ENV_DISK_TIMEOUT}ms`); - } else if (result.status === 'rejected') { - const reason = result.reason instanceof Error ? result.reason.message : String(result.reason); - console.warn(`[MetricsSubprocess] Environment "${enabledEnvs[index].name}" disk check failed: ${reason}`); - } - }); - - // Clean up stale lastDiskWarning entries for deleted environments - const activeEnvIds = new Set(environments.map((e) => e.id)); - for (const envId of lastDiskWarning.keys()) { - if (!activeEnvIds.has(envId)) lastDiskWarning.delete(envId); - } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(`[MetricsSubprocess] Disk space check error: ${message}`); - send({ type: 'error', message: `Disk space check error: ${message}` }); - } -} - -/** - * Handle commands from main process - */ -function handleCommand(command: MainProcessCommand): void { - switch (command.type) { - case 'refresh_environments': - console.log('[MetricsSubprocess] Refreshing environments...'); - // The next collection cycle will pick up the new environments - break; - - case 'update_interval': - console.log(`[MetricsSubprocess] Updating collection interval to ${command.intervalMs}ms`); - COLLECT_INTERVAL = command.intervalMs; - // Clear existing interval and restart with new timing - if (collectInterval) { - clearInterval(collectInterval); - collectInterval = setInterval(collectMetrics, COLLECT_INTERVAL); - } - break; - - case 'shutdown': - console.log('[MetricsSubprocess] Shutdown requested'); - shutdown(); - break; - } -} - -/** - * Graceful shutdown - */ -function shutdown(): void { - isShuttingDown = true; - - if (collectInterval) { - clearInterval(collectInterval); - collectInterval = null; - } - if (diskCheckInterval) { - clearInterval(diskCheckInterval); - diskCheckInterval = null; - } - - lastDiskWarning.clear(); - console.log('[MetricsSubprocess] Stopped'); - process.exit(0); -} - -/** - * Start the metrics collector - */ -async function start(): Promise { - // Load interval from settings - try { - COLLECT_INTERVAL = await getMetricsCollectionInterval(); - console.log(`[MetricsSubprocess] Starting metrics collection (every ${COLLECT_INTERVAL / 1000}s)...`); - } catch (error) { - console.error('[MetricsSubprocess] Failed to load interval from settings, using default 30s'); - COLLECT_INTERVAL = 30000; - } - - // Initial collection - collectMetrics(); - - // Schedule regular collection - collectInterval = setInterval(collectMetrics, COLLECT_INTERVAL); - - // Start disk space checking (every 5 minutes) - can be disabled for Synology NAS - const skipDfCollection = process.env.SKIP_DF_COLLECTION === 'true' || process.env.SKIP_DF_COLLECTION === '1'; - if (!skipDfCollection) { - console.log('[MetricsSubprocess] Starting disk space monitoring (every 5 minutes)'); - checkDiskSpace(); // Initial check - diskCheckInterval = setInterval(checkDiskSpace, DISK_CHECK_INTERVAL); - } else { - console.log('[MetricsSubprocess] Disk space monitoring disabled (SKIP_DF_COLLECTION=true)'); - } - - // Start memory diagnostics logging (every 5 minutes) - setInterval(() => { - const mem = process.memoryUsage(); - console.log( - `[MetricsSubprocess] Memory: heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB, ` + - `rss=${Math.round(mem.rss / 1024 / 1024)}MB` - ); - }, 5 * 60 * 1000); - - // Listen for commands from main process - process.on('message', (message: MainProcessCommand) => { - handleCommand(message); - }); - - // Handle termination signals - process.on('SIGTERM', shutdown); - process.on('SIGINT', shutdown); - - // Signal ready - send({ type: 'ready' }); - - console.log('[MetricsSubprocess] Started successfully'); -} - -// Start the subprocess -start(); diff --git a/src/lib/stores/containers.ts b/src/lib/stores/containers.ts new file mode 100644 index 0000000..17e5067 --- /dev/null +++ b/src/lib/stores/containers.ts @@ -0,0 +1,346 @@ +import { writable, get } from 'svelte/store'; +import { browser } from '$app/environment'; +import type { ContainerInfo, ContainerStats } from '$lib/types'; +import { appendEnvParam, clearStaleEnvironment, environments } from '$lib/stores/environment'; +import { toast } from 'svelte-sonner'; + +export interface AutoUpdateSetting { + enabled: boolean; + label: string; + tooltip: string; + vulnerabilityCriteria?: string; +} + +export interface ContainerStoreState { + /** Container list */ + data: ContainerInfo[]; + /** Live stats keyed by container ID */ + stats: Map; + /** Previous stats snapshot for change detection */ + previousStats: Map; + /** Auto-update settings keyed by container name */ + autoUpdateSettings: Map; + /** Container IDs with pending updates */ + pendingUpdateIds: string[]; + /** Container names for pending updates, keyed by ID */ + pendingUpdateNames: Map; + /** Whether the current environment has vulnerability scanning */ + envHasScanning: boolean; + /** Environment-level vulnerability criteria */ + envVulnerabilityCriteria: 'never' | 'any' | 'critical_high' | 'critical' | 'more_than_current'; + /** True during initial load (no cached data for this env) */ + loading: boolean; + /** The environment ID this data belongs to */ + envId: number | null; +} + +const INITIAL_STATE: ContainerStoreState = { + data: [], + stats: new Map(), + previousStats: new Map(), + autoUpdateSettings: new Map(), + pendingUpdateIds: [], + pendingUpdateNames: new Map(), + envHasScanning: false, + envVulnerabilityCriteria: 'never', + loading: true, + envId: null +}; + +function createContainerStore() { + const { subscribe, set, update } = writable({ ...INITIAL_STATE }); + + // In-flight request tracking to avoid duplicate concurrent fetches + let fetchingContainers = false; + let fetchingStats = false; + + function patch(partial: Partial) { + update((s) => ({ ...s, ...partial })); + } + + function formatSchedule( + scheduleType: string, + cronExpression: string + ): { label: string; tooltip: string } { + if (!cronExpression) return { label: 'on', tooltip: 'Auto-update enabled' }; + + const parts = cronExpression.split(' '); + if (parts.length < 5) return { label: 'cron', tooltip: cronExpression }; + + const [min, hr, , , dow] = parts; + const hourNum = parseInt(hr); + const minNum = parseInt(min); + const ampm = hourNum >= 12 ? 'PM' : 'AM'; + const hour12 = hourNum === 0 ? 12 : hourNum > 12 ? hourNum - 12 : hourNum; + const timeStr = `${hour12}:${minNum.toString().padStart(2, '0')} ${ampm}`; + + if (scheduleType === 'daily' || dow === '*') { + return { label: 'daily', tooltip: `Daily at ${timeStr}` }; + } + + if (scheduleType === 'weekly') { + const days = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']; + const dayName = days[parseInt(dow)] || dow; + return { + label: dayName, + tooltip: `Every ${['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][parseInt(dow)] || dow} at ${timeStr}` + }; + } + + return { label: 'cron', tooltip: cronExpression }; + } + + async function checkScannerSettings(envId: number | null) { + if (!envId) { + patch({ envHasScanning: false, envVulnerabilityCriteria: 'never' }); + return; + } + try { + const [scannerResponse, updateCheckResponse] = await Promise.all([ + fetch(`/api/settings/scanner?env=${envId}&settingsOnly=true`), + fetch(`/api/environments/${envId}/update-check`) + ]); + + let envHasScanning = false; + let envVulnerabilityCriteria: ContainerStoreState['envVulnerabilityCriteria'] = 'never'; + + if (scannerResponse.ok) { + const data = await scannerResponse.json(); + const settings = data.settings || data; + envHasScanning = settings.scanner !== 'none'; + } + + if (updateCheckResponse.ok) { + const data = await updateCheckResponse.json(); + envVulnerabilityCriteria = data.settings?.vulnerabilityCriteria || 'never'; + } + + patch({ envHasScanning, envVulnerabilityCriteria }); + } catch { + patch({ envHasScanning: false, envVulnerabilityCriteria: 'never' }); + } + } + + async function fetchAutoUpdateSettings(envId: number | null) { + const settings = new Map(); + const envParam = envId ? `?env=${envId}` : ''; + + await checkScannerSettings(envId); + + try { + const response = await fetch(`/api/auto-update${envParam}`); + if (response.ok) { + const data = await response.json(); + for (const [containerName, setting] of Object.entries(data)) { + if ( + setting && + typeof setting === 'object' && + 'enabled' in setting && + (setting as any).enabled + ) { + const s = setting as { + enabled: boolean; + scheduleType: string; + cronExpression: string | null; + vulnerabilityCriteria: string; + }; + const { label, tooltip } = formatSchedule( + s.scheduleType, + s.cronExpression || '' + ); + settings.set(containerName, { + enabled: true, + label, + tooltip, + vulnerabilityCriteria: s.vulnerabilityCriteria || 'never' + }); + } + } + } + } catch (err) { + console.error('Failed to fetch auto-update settings:', err); + } + + patch({ autoUpdateSettings: settings }); + } + + async function fetchContainersInternal(envId: number | null) { + if (!browser || !envId || fetchingContainers) return; + fetchingContainers = true; + + const state = get({ subscribe }); + // Only show loading spinner if we have no cached data for this env + const showLoading = state.data.length === 0 || state.envId !== envId; + if (showLoading) { + patch({ loading: true }); + } + + try { + const response = await fetch(appendEnvParam('/api/containers', envId)); + if (!response.ok) { + if (response.status === 404 && envId) { + clearStaleEnvironment(envId); + environments.refresh(); + return; + } + toast.error('Failed to load containers'); + return; + } + const data: ContainerInfo[] = await response.json(); + patch({ data, envId }); + + // Fetch auto-update settings after containers load + await fetchAutoUpdateSettings(envId); + } catch (error) { + console.error('Failed to fetch containers:', error); + toast.error('Failed to load containers'); + } finally { + patch({ loading: false }); + fetchingContainers = false; + } + } + + let statsAbortController: AbortController | null = null; + + async function fetchStatsInternal(envId: number | null) { + if (!browser || !envId || fetchingStats) return; + fetchingStats = true; + + // Abort any previous in-flight stream + statsAbortController?.abort(); + statsAbortController = new AbortController(); + + // Snapshot previous stats once at cycle start + update((s) => ({ ...s, previousStats: new Map(s.stats) })); + + try { + const response = await fetch( + appendEnvParam('/api/containers/stats/stream', envId), + { signal: statsAbortController.signal } + ); + + if (!response.ok || !response.body) { + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + let currentEvent = ''; + for (const line of lines) { + if (line.startsWith(':')) continue; + + if (line.startsWith('event: ')) { + currentEvent = line.slice(7).trim(); + } else if (line.startsWith('data: ')) { + if (currentEvent === 'stat') { + try { + const stat: ContainerStats = JSON.parse(line.slice(6)); + // Merge into existing stats map + update((s) => { + const newStats = new Map(s.stats); + newStats.set(stat.id, stat); + return { ...s, stats: newStats }; + }); + } catch { + // Skip malformed data + } + } + // done/error events — just let the loop finish + currentEvent = ''; + } + } + } + } catch (error: any) { + if (error?.name !== 'AbortError') { + console.error('Failed to fetch container stats:', error); + } + } finally { + fetchingStats = false; + } + } + + async function loadPendingUpdatesInternal(envId: number | null) { + if (!browser || !envId) return; + try { + const response = await fetch(appendEnvParam('/api/containers/pending-updates', envId)); + if (!response.ok) return; + const data = await response.json(); + if (data.pendingUpdates && data.pendingUpdates.length > 0) { + patch({ + pendingUpdateIds: data.pendingUpdates.map((u: any) => u.containerId), + pendingUpdateNames: new Map( + data.pendingUpdates.map((u: any) => [u.containerId, u.containerName]) + ) + }); + } + } catch { + // Ignore errors - background load + } + } + + return { + subscribe, + + /** Full refresh: containers + auto-update settings */ + refreshContainers(envId: number | null) { + return fetchContainersInternal(envId); + }, + + /** Stats-only refresh (called on 5s interval) */ + refreshStats(envId: number | null) { + return fetchStatsInternal(envId); + }, + + /** Full refresh: containers + stats + settings + pending updates */ + async refresh(envId: number | null) { + await Promise.all([ + fetchContainersInternal(envId), + fetchStatsInternal(envId), + loadPendingUpdatesInternal(envId) + ]); + }, + + /** Reload pending updates from database */ + loadPendingUpdates(envId: number | null) { + return loadPendingUpdatesInternal(envId); + }, + + /** Clear all data (environment switch) */ + invalidate() { + statsAbortController?.abort(); + fetchingStats = false; + set({ + ...INITIAL_STATE, + loading: true + }); + }, + + /** Clear data without loading state (no environment selected) */ + clear() { + statsAbortController?.abort(); + fetchingStats = false; + set({ ...INITIAL_STATE, loading: false }); + }, + + /** Update pending update IDs and names directly (from check-updates action) */ + setPendingUpdates(ids: string[], names: Map) { + patch({ pendingUpdateIds: ids, pendingUpdateNames: names }); + }, + + /** Patch arbitrary fields */ + patch + }; +} + +export const containerStore = createContainerStore(); diff --git a/src/lib/utils/pem.ts b/src/lib/utils/pem.ts index 3a82b01..b3fe274 100644 --- a/src/lib/utils/pem.ts +++ b/src/lib/utils/pem.ts @@ -1,6 +1,6 @@ /** * Clean PEM content by removing whitespace artifacts from copy/paste. - * Bun's TLS is strict about PEM format - it fails when certificates have + * TLS implementations are strict about PEM format - they fail when certificates have * leading/trailing spaces on lines or extra blank lines. * * @param pem - The PEM content to clean diff --git a/src/lib/utils/sse-fetch.ts b/src/lib/utils/sse-fetch.ts new file mode 100644 index 0000000..54c38bb --- /dev/null +++ b/src/lib/utils/sse-fetch.ts @@ -0,0 +1,73 @@ +import type { JobLine } from '$lib/server/jobs'; + +/** + * Reads a job-based response (POST returns { jobId }) and polls until complete. + * Drop-in replacement for readSSEResponse when the endpoint has been migrated to jobs. + * + * Returns the job's final result (equivalent to the 'result' event data in SSE). + */ +export async function readJobResponse( + response: Response +): Promise<{ success?: boolean; error?: string; [key: string]: unknown }> { + // Fall through for non-JSON or error responses + if (!response.ok) { + try { + return await response.json(); + } catch { + return { success: false, error: `HTTP ${response.status}` }; + } + } + + const data = await response.json(); + + // If the response is a { jobId } shape, poll the job endpoint + if (data && typeof data === 'object' && 'jobId' in data) { + const result = await watchJob(data.jobId as string, () => { + // readJobResponse callers don't need line-by-line updates + }); + return result as { success?: boolean; error?: string; [key: string]: unknown }; + } + + // Fallback: response was already the final result (e.g. application/json sync path) + return data; +} + +const POLL_INTERVAL_MS = 500; + +/** + * Polls /api/jobs/{jobId} every 500ms. Calls onLine for each new line as they arrive. + * Resolves with the job's final result when status is 'done' or 'error'. + */ +export async function watchJob( + jobId: string, + onLine: (line: JobLine) => void +): Promise { + let cursor = 0; + + while (true) { + const res = await fetch(`/api/jobs/${jobId}`); + if (!res.ok) { + throw new Error(`Job poll failed: HTTP ${res.status}`); + } + + const job = await res.json() as { + id: string; + status: 'running' | 'done' | 'error'; + lines: JobLine[]; + result: unknown; + }; + + // Deliver new lines since last poll + const newLines = job.lines.slice(cursor); + cursor = job.lines.length; + for (const line of newLines) { + onLine(line); + } + + if (job.status !== 'running') { + return job.result; + } + + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + } +} diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 27bc5c8..50a80f0 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -920,6 +920,7 @@ } unsubscribeDashboardData(); unsubscribePrefs(); + mobileWatcher.destroy(); }); diff --git a/src/routes/activity/+page.svelte b/src/routes/activity/+page.svelte index 9d2b302..f821ca4 100644 --- a/src/routes/activity/+page.svelte +++ b/src/routes/activity/+page.svelte @@ -521,16 +521,12 @@ // Add to beginning of events (prepend new events) - use Set for fast duplicate check if (!eventIds.has(newEvent.id)) { eventIds.add(newEvent.id); - // Use unshift() for in-place mutation instead of spread for O(n) copy - events.unshift(newEvent); - events = events; // Trigger Svelte reactivity + events = [newEvent, ...events]; total = total + 1; // Add container to list if not already there if (newEvent.containerName && !containers.includes(newEvent.containerName)) { - containers.push(newEvent.containerName); - containers.sort(); - containers = containers; // Trigger Svelte reactivity + containers = [...containers, newEvent.containerName].sort(); } } } catch { @@ -624,6 +620,11 @@ }).then(() => { connectSSE(); initialLoadDone = true; + }).catch((err) => { + console.error('[Activity] Init chain failed:', err); + // Connect SSE anyway so live events still work + connectSSE(); + initialLoadDone = true; }); // Note: In Svelte 5, cleanup must be in onDestroy, not returned from onMount }); diff --git a/src/routes/api/activity/events/+server.ts b/src/routes/api/activity/events/+server.ts index 392c915..8226dc4 100644 --- a/src/routes/api/activity/events/+server.ts +++ b/src/routes/api/activity/events/+server.ts @@ -3,6 +3,7 @@ import { containerEventEmitter } from '$lib/server/event-collector'; import { authorize } from '$lib/server/authorize'; import { json } from '@sveltejs/kit'; + export const GET: RequestHandler = async ({ cookies }) => { const auth = await authorize(cookies); @@ -52,6 +53,7 @@ export const GET: RequestHandler = async ({ cookies }) => { }; // Send initial connection event + sendEvent('connected', { timestamp: new Date().toISOString() }); // Send heartbeat to keep connection alive (every 5s to prevent Traefik 10s idle timeout) @@ -84,6 +86,7 @@ export const GET: RequestHandler = async ({ cookies }) => { }, cancel() { // Cleanup when client disconnects + clearInterval(heartbeatInterval); if (handleEvent) { containerEventEmitter.off('event', handleEvent); diff --git a/src/routes/api/auth/login/+server.ts b/src/routes/api/auth/login/+server.ts index 6468283..9ce6787 100644 --- a/src/routes/api/auth/login/+server.ts +++ b/src/routes/api/auth/login/+server.ts @@ -30,11 +30,14 @@ export const POST: RequestHandler = async (event) => { } // Rate limiting by IP and username - const clientIp = getClientAddress(); + const clientIp = request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() + || request.headers.get('x-real-ip') + || getClientAddress(); const rateLimitKey = `${clientIp}:${username}`; const { limited, retryAfter } = isRateLimited(rateLimitKey); if (limited) { + console.warn(`[Auth] Login rate-limited: user=${username} ip=${clientIp} retryAfter=${retryAfter}s`); return json( { error: `Too many login attempts. Please try again in ${retryAfter} seconds.` }, { status: 429 } @@ -66,6 +69,7 @@ export const POST: RequestHandler = async (event) => { if (!result.success) { recordFailedAttempt(rateLimitKey); + console.warn(`[Auth] Login failed: user=${username} provider=${authProviderType} ip=${clientIp} reason=${result.error || 'Authentication failed'}`); return json({ error: result.error || 'Authentication failed' }, { status: 401 }); } @@ -80,12 +84,14 @@ export const POST: RequestHandler = async (event) => { const user = await getUserByUsername(username); if (!user || !(await verifyMfaToken(user.id, mfaToken))) { recordFailedAttempt(rateLimitKey); + console.warn(`[Auth] MFA failed: user=${username} ip=${clientIp}`); return json({ error: 'Invalid MFA code' }, { status: 401 }); } // MFA verified, create session - const session = await createUserSession(user.id, authProviderType, cookies); + const session = await createUserSession(user.id, authProviderType, cookies, request); clearRateLimit(rateLimitKey); + console.log(`[Auth] Login successful: user=${username} provider=${authProviderType} ip=${clientIp} mfa=yes`); // Audit log await auditAuth(event, 'login', user.username, { @@ -107,8 +113,9 @@ export const POST: RequestHandler = async (event) => { // No MFA, create session directly if (result.user) { - const session = await createUserSession(result.user.id, authProviderType, cookies); + const session = await createUserSession(result.user.id, authProviderType, cookies, request); clearRateLimit(rateLimitKey); + console.log(`[Auth] Login successful: user=${result.user.username} provider=${authProviderType} ip=${clientIp} mfa=no`); // Audit log await auditAuth(event, 'login', result.user.username, { diff --git a/src/routes/api/auth/logout/+server.ts b/src/routes/api/auth/logout/+server.ts index a82adf0..e09d589 100644 --- a/src/routes/api/auth/logout/+server.ts +++ b/src/routes/api/auth/logout/+server.ts @@ -11,8 +11,12 @@ export const POST: RequestHandler = async (event) => { // Get current user before destroying session for audit log const auth = await authorize(cookies); const username = auth.user?.username || 'unknown'; + const clientIp = event.request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() + || event.request.headers.get('x-real-ip') + || event.getClientAddress(); await destroySession(cookies); + console.log(`[Auth] Logout: user=${username} ip=${clientIp}`); // Audit log await auditAuth(event, 'logout', username); diff --git a/src/routes/api/auth/oidc/callback/+server.ts b/src/routes/api/auth/oidc/callback/+server.ts index d20a536..a6ffb11 100644 --- a/src/routes/api/auth/oidc/callback/+server.ts +++ b/src/routes/api/auth/oidc/callback/+server.ts @@ -17,9 +17,15 @@ export const GET: RequestHandler = async (event) => { const error = url.searchParams.get('error'); const errorDescription = url.searchParams.get('error_description'); + // Extract client IP for logging + const clientIp = event.request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() + || event.request.headers.get('x-real-ip') + || event.getClientAddress(); + // Handle error from IdP if (error) { console.error('OIDC error from IdP:', error, errorDescription); + console.warn(`[Auth] OIDC login failed: ip=${clientIp} error=${errorDescription || error}`); const errorMsg = encodeURIComponent(errorDescription || error); throw redirect(302, `/login?error=${errorMsg}`); } @@ -33,12 +39,14 @@ export const GET: RequestHandler = async (event) => { const result = await handleOidcCallback(code, state); if (!result.success || !result.user) { + console.warn(`[Auth] OIDC login failed: ip=${clientIp} error=${result.error || 'Authentication failed'}`); const errorMsg = encodeURIComponent(result.error || 'Authentication failed'); throw redirect(302, `/login?error=${errorMsg}`); } // Create session - await createUserSession(result.user.id, 'oidc', cookies); + await createUserSession(result.user.id, 'oidc', cookies, event.request); + console.log(`[Auth] OIDC login successful: user=${result.user.username} provider=${result.providerName || 'oidc'} ip=${clientIp}`); // Audit log await auditAuth(event, 'login', result.user.username, { diff --git a/src/routes/api/batch/+server.ts b/src/routes/api/batch/+server.ts index 234ff45..0c7757c 100644 --- a/src/routes/api/batch/+server.ts +++ b/src/routes/api/batch/+server.ts @@ -8,8 +8,6 @@ import { pauseContainer, unpauseContainer, removeContainer, - inspectContainer, - listContainers, removeImage, removeVolume, removeNetwork @@ -23,6 +21,8 @@ import { } from '$lib/server/stacks'; import { deleteAutoUpdateSchedule, getAutoUpdateSetting, removePendingContainerUpdate } from '$lib/server/db'; import { unregisterSchedule } from '$lib/server/scheduler'; +import { prefersJSON } from '$lib/server/sse'; +import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs'; // SSE Event types export type BatchEventType = 'start' | 'progress' | 'complete' | 'error'; @@ -105,15 +105,13 @@ interface BatchRequest { async function processWithConcurrency( items: T[], concurrency: number, - processor: (item: T, index: number) => Promise, - signal: AbortSignal + processor: (item: T, index: number) => Promise ): Promise { let currentIndex = 0; const total = items.length; async function processNext(): Promise { while (currentIndex < total) { - if (signal.aborted) return; const index = currentIndex++; await processor(items[index], index); } @@ -128,7 +126,7 @@ async function processWithConcurrency( } /** - * Unified batch operations endpoint with SSE streaming. + * Unified batch operations endpoint (job pattern). * Handles bulk operations for containers, images, volumes, networks, and stacks. */ export const POST: RequestHandler = async ({ url, cookies, request }) => { @@ -182,124 +180,74 @@ export const POST: RequestHandler = async ({ url, cookies, request }) => { // Check if audit is needed (enterprise only) const needsAudit = auth.isEnterprise; - // Create abort controller for cancellation - const abortController = new AbortController(); + // Sync path for API clients that prefer plain JSON (Accept: application/json only) + if (prefersJSON(request)) { + let successCount = 0; + let failCount = 0; - const encoder = new TextEncoder(); - let controllerClosed = false; - let keepaliveInterval: ReturnType | null = null; + await processWithConcurrency(items, 3, async (item, index) => { + const { id, name } = item; + try { + await executeOperation(entityType, operation, id, name, envIdNum, options, needsAudit); + successCount++; + } catch { + failCount++; + } + }); - const stream = new ReadableStream({ - async start(controller) { - const safeEnqueue = (data: BatchEvent) => { - if (!controllerClosed) { - try { - controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`)); - } catch { - controllerClosed = true; - abortController.abort(); - } - } - }; + return json({ + type: 'complete', + summary: { total: items.length, success: successCount, failed: failCount } + }); + } - // Send SSE keepalive comments every 5s - keepaliveInterval = setInterval(() => { - if (controllerClosed) return; - try { - controller.enqueue(encoder.encode(`: keepalive\n\n`)); - } catch { - controllerClosed = true; - abortController.abort(); - } - }, 5000); + // Job pattern: create job, process in background, return jobId immediately + const job = createJob(); - let successCount = 0; - let failCount = 0; + (async () => { + let successCount = 0; + let failCount = 0; - // Send start event - safeEnqueue({ - type: 'start', - total: items.length + appendLine(job, { data: { type: 'start', total: items.length } }); + + await processWithConcurrency(items, 3, async (item, index) => { + const { id, name } = item; + + appendLine(job, { + data: { type: 'progress', id, name, status: 'processing', current: index + 1, total: items.length } }); - // Process items with concurrency of 3 - await processWithConcurrency( - items, - 3, - async (item, index) => { - if (abortController.signal.aborted) return; - - const { id, name } = item; - - // Send processing status - safeEnqueue({ + try { + await executeOperation(entityType, operation, id, name, envIdNum, options, needsAudit); + appendLine(job, { + data: { type: 'progress', id, name, status: 'success', current: index + 1, total: items.length } + }); + successCount++; + } catch (error: any) { + appendLine(job, { + data: { type: 'progress', id, name, - status: 'processing', + status: 'error', + error: error.message || 'Unknown error', current: index + 1, total: items.length - }); - - try { - await executeOperation(entityType, operation, id, name, envIdNum, options, needsAudit); - - safeEnqueue({ - type: 'progress', - id, - name, - status: 'success', - current: index + 1, - total: items.length - }); - successCount++; - } catch (error: any) { - safeEnqueue({ - type: 'progress', - id, - name, - status: 'error', - error: error.message || 'Unknown error', - current: index + 1, - total: items.length - }); - failCount++; } - }, - abortController.signal - ); - - // Send complete event - safeEnqueue({ - type: 'complete', - summary: { - total: items.length, - success: successCount, - failed: failCount - } - }); - - if (keepaliveInterval) { - clearInterval(keepaliveInterval); + }); + failCount++; } - controller.close(); - }, - cancel() { - controllerClosed = true; - abortController.abort(); - if (keepaliveInterval) { - clearInterval(keepaliveInterval); - } - } - }); + }); - return new Response(stream, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive' - } - }); + const completeEvent = { + type: 'complete', + summary: { total: items.length, success: successCount, failed: failCount } + }; + appendLine(job, { data: completeEvent }); + completeJob(job, completeEvent); + })().catch((err) => failJob(job, err.message)); + + return json({ jobId: job.id }); }; /** diff --git a/src/routes/api/containers/+server.ts b/src/routes/api/containers/+server.ts index 243e0ca..8c1e170 100644 --- a/src/routes/api/containers/+server.ts +++ b/src/routes/api/containers/+server.ts @@ -1,5 +1,5 @@ import { json } from '@sveltejs/kit'; -import { listContainers, createContainer, pullImage, EnvironmentNotFoundError, type CreateContainerOptions } from '$lib/server/docker'; +import { listContainers, createContainer, pullImage, EnvironmentNotFoundError, DockerConnectionError, type CreateContainerOptions } from '$lib/server/docker'; import { authorize } from '$lib/server/authorize'; import { auditContainer } from '$lib/server/audit'; import { hasEnvironments } from '$lib/server/db'; @@ -40,7 +40,9 @@ export const GET: RequestHandler = async ({ url, cookies }) => { if (error instanceof EnvironmentNotFoundError) { return json({ error: 'Environment not found' }, { status: 404 }); } - console.error('Error listing containers:', error); + if (!(error instanceof DockerConnectionError)) { + console.error('Error listing containers:', error); + } // Return empty array instead of error to allow UI to load return json([]); } diff --git a/src/routes/api/containers/[id]/files/download/+server.ts b/src/routes/api/containers/[id]/files/download/+server.ts index b271f75..4bcf2b3 100644 --- a/src/routes/api/containers/[id]/files/download/+server.ts +++ b/src/routes/api/containers/[id]/files/download/+server.ts @@ -1,3 +1,4 @@ +import { gzipSync } from 'node:zlib'; import { getContainerArchive, statContainerPath } from '$lib/server/docker'; import { authorize } from '$lib/server/authorize'; import type { RequestHandler } from './$types'; @@ -50,9 +51,9 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { let extension = '.tar'; if (format === 'tar.gz') { - // Compress with gzip using Bun's native implementation + // Compress with gzip const tarData = new Uint8Array(await response.arrayBuffer()); - body = Bun.gzipSync(tarData); + body = gzipSync(tarData); contentType = 'application/gzip'; extension = '.tar.gz'; } diff --git a/src/routes/api/containers/[id]/logs/stream/+server.ts b/src/routes/api/containers/[id]/logs/stream/+server.ts index 2dec81d..b6266aa 100644 --- a/src/routes/api/containers/[id]/logs/stream/+server.ts +++ b/src/routes/api/containers/[id]/logs/stream/+server.ts @@ -1,6 +1,8 @@ import type { RequestHandler } from './$types'; import { authorize } from '$lib/server/authorize'; import { getEnvironment } from '$lib/server/db'; +import { unixSocketRequest, unixSocketStreamRequest, httpsAgentRequest } from '$lib/server/docker'; +import type { DockerClientConfig as BaseDockerClientConfig } from '$lib/server/docker'; import { sendEdgeRequest, sendEdgeStreamRequest, isEdgeConnected } from '$lib/server/hawser'; import { existsSync } from 'node:fs'; import { homedir } from 'node:os'; @@ -56,6 +58,7 @@ async function getDockerConfig(envId?: number | null): Promise { @@ -226,6 +230,7 @@ async function handleEdgeLogsStream(containerId: string, tail: string, environme cancelStream = cancel; }, cancel() { + controllerClosed = true; if (heartbeatInterval) { clearInterval(heartbeatInterval); @@ -279,28 +284,16 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { let inspectResponse: Response; if (config.type === 'socket') { - inspectResponse = await fetch(`http://localhost${inspectPath}`, { - // @ts-ignore - Bun supports unix socket - unix: config.socketPath - }); + inspectResponse = await unixSocketRequest(config.socketPath, inspectPath); + } else if (config.type === 'https') { + const extraHeaders: Record = {}; + if (config.hawserToken) extraHeaders['X-Hawser-Token'] = config.hawserToken; + inspectResponse = await httpsAgentRequest(config as BaseDockerClientConfig, inspectPath, {}, false, extraHeaders); } else { - const inspectUrl = `${config.type}://${config.host}:${config.port}${inspectPath}`; + const inspectUrl = `http://${config.host}:${config.port}${inspectPath}`; const inspectHeaders: Record = {}; if (config.hawserToken) inspectHeaders['X-Hawser-Token'] = config.hawserToken; - const fetchOpts: any = { headers: inspectHeaders }; - if (config.type === 'https') { - fetchOpts.tls = { - sessionTimeout: 0, - servername: config.host, - rejectUnauthorized: !config.skipVerify - }; - if (config.ca) fetchOpts.tls.ca = [config.ca]; - if (config.cert) fetchOpts.tls.cert = [config.cert]; - if (config.key) fetchOpts.tls.key = config.key; - fetchOpts.keepalive = false; - if (process.env.DEBUG_TLS) fetchOpts.verbose = true; - } - inspectResponse = await fetch(inspectUrl, fetchOpts); + inspectResponse = await fetch(inspectUrl, { headers: inspectHeaders }); } if (inspectResponse.ok) { @@ -322,6 +315,7 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { const stream = new ReadableStream({ async start(controller) { + const encoder = new TextEncoder(); const safeEnqueue = (data: string) => { @@ -343,32 +337,16 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { let response: Response; if (config.type === 'socket') { - response = await fetch(`http://localhost${logsPath}`, { - // @ts-ignore - Bun supports unix socket - unix: config.socketPath, - signal: abortController?.signal - }); + response = await unixSocketStreamRequest(config.socketPath, logsPath); + } else if (config.type === 'https') { + const extraHeaders: Record = {}; + if (config.hawserToken) extraHeaders['X-Hawser-Token'] = config.hawserToken; + response = await httpsAgentRequest(config as BaseDockerClientConfig, logsPath, {}, true, extraHeaders); } else { - const logsUrl = `${config.type}://${config.host}:${config.port}${logsPath}`; + const logsUrl = `http://${config.host}:${config.port}${logsPath}`; const logsHeaders: Record = {}; if (config.hawserToken) logsHeaders['X-Hawser-Token'] = config.hawserToken; - const fetchOpts: any = { - headers: logsHeaders, - signal: abortController?.signal - }; - if (config.type === 'https') { - fetchOpts.tls = { - sessionTimeout: 0, - servername: config.host, - rejectUnauthorized: !config.skipVerify - }; - if (config.ca) fetchOpts.tls.ca = [config.ca]; - if (config.cert) fetchOpts.tls.cert = [config.cert]; - if (config.key) fetchOpts.tls.key = config.key; - fetchOpts.keepalive = false; - if (process.env.DEBUG_TLS) fetchOpts.verbose = true; - } - response = await fetch(logsUrl, fetchOpts); + response = await fetch(logsUrl, { headers: logsHeaders }); } if (!response.ok) { @@ -434,7 +412,8 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { } } - reader.releaseLock(); + await reader.cancel().catch(() => {}); + reader.releaseLock(); } catch (error) { if (!controllerClosed) { const errorMsg = error instanceof Error ? error.message : String(error); @@ -444,7 +423,14 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { } } + // Clean up on normal stream end (not just cancel) + if (heartbeatInterval) { + clearInterval(heartbeatInterval); + heartbeatInterval = null; + } if (!controllerClosed) { + controllerClosed = true; + try { controller.close(); } catch { @@ -453,7 +439,10 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { } }, cancel() { - controllerClosed = true; + if (!controllerClosed) { + controllerClosed = true; + + } if (heartbeatInterval) { clearInterval(heartbeatInterval); heartbeatInterval = null; diff --git a/src/routes/api/containers/batch-update-stream/+server.ts b/src/routes/api/containers/batch-update-stream/+server.ts index b076c4c..c7fd17b 100644 --- a/src/routes/api/containers/batch-update-stream/+server.ts +++ b/src/routes/api/containers/batch-update-stream/+server.ts @@ -16,6 +16,7 @@ import { getScannerSettings, scanImage } from '$lib/server/scanner'; import { saveVulnerabilityScan, removePendingContainerUpdate, type VulnerabilityCriteria } from '$lib/server/db'; import { parseImageNameAndTag, shouldBlockUpdate, combineScanSummaries, isDockhandContainer } from '$lib/server/scheduler/tasks/update-utils'; import { recreateContainer } from '$lib/server/scheduler/tasks/container-update'; +import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs'; export interface ScanResult { critical: number; @@ -96,423 +97,123 @@ export const POST: RequestHandler = async (event) => { return json({ error: 'containerIds array is required' }, { status: 400 }); } - const encoder = new TextEncoder(); - let controllerClosed = false; - let keepaliveInterval: ReturnType | null = null; + // Job pattern: create job, run in background, return jobId immediately + const job = createJob(); - const stream = new ReadableStream({ - async start(controller) { - const safeEnqueue = (data: UpdateProgress) => { - if (!controllerClosed) { - try { - controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`)); - } catch { - controllerClosed = true; - } - } - }; + const sendData = (data: UpdateProgress) => { + appendLine(job, { data }); + }; - // Send SSE keepalive comments every 5s to prevent Traefik (10s idle timeout) from closing connection - keepaliveInterval = setInterval(() => { - if (controllerClosed) return; - try { - controller.enqueue(encoder.encode(`: keepalive\n\n`)); - } catch { - controllerClosed = true; - } - }, 5000); + (async () => { + let successCount = 0; + let failCount = 0; + let blockedCount = 0; + let skippedCount = 0; - let successCount = 0; - let failCount = 0; - let blockedCount = 0; - let skippedCount = 0; + // Get scanner settings for this environment + const scannerSettings = await getScannerSettings(envIdNum); + // Scan if scanning is enabled (scanner !== 'none') + // The vulnerabilityCriteria only controls whether to BLOCK updates, not whether to SCAN + const shouldScan = scannerSettings.scanner !== 'none'; - // Get scanner settings for this environment - const scannerSettings = await getScannerSettings(envIdNum); - // Scan if scanning is enabled (scanner !== 'none') - // The vulnerabilityCriteria only controls whether to BLOCK updates, not whether to SCAN - const shouldScan = scannerSettings.scanner !== 'none'; + // Send start event + sendData({ + type: 'start', + total: containerIds.length, + message: `Starting update of ${containerIds.length} container${containerIds.length > 1 ? 's' : ''}${shouldScan ? ' with vulnerability scanning' : ''}` + }); - // Send start event - safeEnqueue({ - type: 'start', - total: containerIds.length, - message: `Starting update of ${containerIds.length} container${containerIds.length > 1 ? 's' : ''}${shouldScan ? ' with vulnerability scanning' : ''}` - }); + // Process containers sequentially + for (let i = 0; i < containerIds.length; i++) { + const containerId = containerIds[i]; + let containerName = 'unknown'; - // Process containers sequentially - for (let i = 0; i < containerIds.length; i++) { - const containerId = containerIds[i]; - let containerName = 'unknown'; + try { + // Find container + const containers = await listContainers(true, envIdNum); + const container = containers.find(c => c.id === containerId); - try { - // Find container - const containers = await listContainers(true, envIdNum); - const container = containers.find(c => c.id === containerId); - - if (!container) { - safeEnqueue({ - type: 'progress', - containerId, - containerName: 'unknown', - step: 'failed', - current: i + 1, - total: containerIds.length, - success: false, - error: 'Container not found' - }); - failCount++; - continue; - } - - containerName = container.name; - - // Get full container config - const inspectData = await inspectContainer(containerId, envIdNum) as any; - const wasRunning = inspectData.State.Running; - const config = inspectData.Config; - const hostConfig = inspectData.HostConfig; - const imageName = config.Image; - const currentImageId = inspectData.Image; - - // Skip Dockhand container - cannot update itself - if (isDockhandContainer(imageName)) { - safeEnqueue({ - type: 'progress', - containerId, - containerName, - step: 'skipped', - current: i + 1, - total: containerIds.length, - success: true, - message: `Skipping ${containerName} - cannot update Dockhand itself` - }); - skippedCount++; - continue; - } - - // Skip digest-pinned images - they are explicitly locked to a specific version - if (isDigestBasedImage(imageName)) { - safeEnqueue({ - type: 'progress', - containerId, - containerName, - step: 'skipped', - current: i + 1, - total: containerIds.length, - success: true, - message: `Skipping ${containerName} - image pinned to specific digest` - }); - skippedCount++; - continue; - } - - // Step 1: Pull latest image - safeEnqueue({ + if (!container) { + sendData({ type: 'progress', containerId, - containerName, - step: 'pulling', + containerName: 'unknown', + step: 'failed', current: i + 1, total: containerIds.length, - message: `Pulling ${imageName}...` + success: false, + error: 'Container not found' }); + failCount++; + continue; + } - try { - await pullImage(imageName, (data: any) => { - if (data.status) { - safeEnqueue({ - type: 'pull_log', - containerId, - containerName, - pullStatus: data.status, - pullId: data.id, - pullProgress: data.progress - }); - } - }, envIdNum); - } catch (pullError: any) { - safeEnqueue({ - type: 'progress', - containerId, - containerName, - step: 'failed', - current: i + 1, - total: containerIds.length, - success: false, - error: `Pull failed: ${pullError.message}` - }); - failCount++; - continue; - } + containerName = container.name; - // SAFE-PULL FLOW with vulnerability scanning - if (shouldScan && !isDigestBasedImage(imageName)) { - const tempTag = getTempImageTag(imageName); + // Get full container config + const inspectData = await inspectContainer(containerId, envIdNum) as any; + const config = inspectData.Config; + const imageName = config.Image; + const currentImageId = inspectData.Image; - // Get new image ID - const newImageId = await getImageIdByTag(imageName, envIdNum); - if (!newImageId) { - safeEnqueue({ - type: 'progress', - containerId, - containerName, - step: 'failed', - current: i + 1, - total: containerIds.length, - success: false, - error: 'Failed to get new image ID after pull' - }); - failCount++; - continue; - } - - // Restore original tag to old image (safety) - const [oldRepo, oldTag] = parseImageNameAndTag(imageName); - try { - await tagImage(currentImageId, oldRepo, oldTag, envIdNum); - } catch { - // Ignore - old image might have been removed - } - - // Tag new image with temp suffix - const [tempRepo, tempTagName] = parseImageNameAndTag(tempTag); - await tagImage(newImageId, tempRepo, tempTagName, envIdNum); - - // Step 2: Scan temp image - safeEnqueue({ - type: 'scan_start', - containerId, - containerName, - step: 'scanning', - current: i + 1, - total: containerIds.length, - message: `Scanning ${imageName} for vulnerabilities...` - }); - - let scanBlocked = false; - let blockReason = ''; - let finalScanResult: ScanResult | undefined; - let individualScannerResults: ScannerResult[] = []; - - try { - const scanResults = await scanImage(tempTag, envIdNum, (progress) => { - if (progress.output || progress.message) { - safeEnqueue({ - type: 'scan_log', - containerId, - containerName, - scanner: progress.scanner, - message: progress.output || progress.message - }); - } - }); - - if (scanResults.length > 0) { - const scanSummary = combineScanSummaries(scanResults); - finalScanResult = { - critical: scanSummary.critical, - high: scanSummary.high, - medium: scanSummary.medium, - low: scanSummary.low, - negligible: scanSummary.negligible, - unknown: scanSummary.unknown - }; - - // Build individual scanner results - individualScannerResults = scanResults.map(result => ({ - scanner: result.scanner as 'grype' | 'trivy', - critical: result.summary.critical, - high: result.summary.high, - medium: result.summary.medium, - low: result.summary.low, - negligible: result.summary.negligible, - unknown: result.summary.unknown - })); - - // Save scan results - for (const result of scanResults) { - try { - await saveVulnerabilityScan({ - environmentId: envIdNum, - imageId: newImageId, - imageName: result.imageName, - scanner: result.scanner, - scannedAt: result.scannedAt, - scanDuration: result.scanDuration, - criticalCount: result.summary.critical, - highCount: result.summary.high, - mediumCount: result.summary.medium, - lowCount: result.summary.low, - negligibleCount: result.summary.negligible, - unknownCount: result.summary.unknown, - vulnerabilities: result.vulnerabilities, - error: result.error ?? null - }); - } catch { /* ignore save errors */ } - } - - // Check if blocked - const { blocked, reason } = shouldBlockUpdate(vulnerabilityCriteria, scanSummary, undefined); - if (blocked) { - scanBlocked = true; - blockReason = reason; - } - } - - // Collect vulnerabilities from all scanners (cap at 100) - const vulnerabilities = scanResults - .flatMap(r => r.vulnerabilities || []) - .slice(0, 100) - .map(v => ({ - id: v.id, - severity: v.severity, - package: v.package, - version: v.version, - fixedVersion: v.fixedVersion, - link: v.link, - scanner: v.scanner - })); - - safeEnqueue({ - type: 'scan_complete', - containerId, - containerName, - scanResult: finalScanResult, - scannerResults: individualScannerResults.length > 0 ? individualScannerResults : undefined, - vulnerabilities: vulnerabilities.length > 0 ? vulnerabilities : undefined, - message: finalScanResult - ? `Scan complete: ${finalScanResult.critical} critical, ${finalScanResult.high} high, ${finalScanResult.medium} medium, ${finalScanResult.low} low` - : 'Scan complete: no vulnerabilities found' - }); - - } catch (scanErr: any) { - safeEnqueue({ - type: 'progress', - containerId, - containerName, - step: 'failed', - current: i + 1, - total: containerIds.length, - success: false, - error: `Scan failed: ${scanErr.message}` - }); - - // Clean up temp image on scan failure - try { - await removeTempImage(newImageId, envIdNum); - } catch { /* ignore cleanup errors */ } - - failCount++; - continue; - } - - if (scanBlocked) { - // BLOCKED - Remove temp image and skip this container - safeEnqueue({ - type: 'blocked', - containerId, - containerName, - step: 'blocked', - current: i + 1, - total: containerIds.length, - success: false, - scanResult: finalScanResult, - scannerResults: individualScannerResults.length > 0 ? individualScannerResults : undefined, - blockReason, - message: `Update blocked: ${blockReason}` - }); - - try { - await removeTempImage(newImageId, envIdNum); - } catch { /* ignore cleanup errors */ } - - blockedCount++; - continue; - } - - // APPROVED - Re-tag to original - await tagImage(newImageId, oldRepo, oldTag, envIdNum); - try { - await removeTempImage(tempTag, envIdNum); - } catch { /* ignore cleanup errors */ } - } - - // Progress logging function for shared functions - const logProgress = (message: string) => { - safeEnqueue({ - type: 'progress', - containerId, - containerName, - step: 'creating', - current: i + 1, - total: containerIds.length, - message - }); - }; - - let updateSuccess = false; - let newContainerId = containerId; - - safeEnqueue({ + // Skip Dockhand container - cannot update itself + if (isDockhandContainer(imageName)) { + sendData({ type: 'progress', containerId, containerName, - step: 'creating', - current: i + 1, - total: containerIds.length, - message: `Recreating ${containerName}...` - }); - - updateSuccess = await recreateContainer(containerName, envIdNum, logProgress, imageName); - if (updateSuccess) { - const updatedContainers = await listContainers(true, envIdNum); - const updatedContainer = updatedContainers.find(c => c.name === containerName); - if (updatedContainer) { - newContainerId = updatedContainer.id; - } - } - - if (!updateSuccess) { - safeEnqueue({ - type: 'progress', - containerId, - containerName, - step: 'failed', - current: i + 1, - total: containerIds.length, - success: false, - error: 'Container recreation failed' - }); - failCount++; - continue; - } - - // Audit log - await auditContainer(event, 'update', newContainerId, containerName, envIdNum, { batchUpdate: true }); - - // Done with this container - use original containerId for UI consistency - safeEnqueue({ - type: 'progress', - containerId, - containerName, - step: 'done', + step: 'skipped', current: i + 1, total: containerIds.length, success: true, - message: `${containerName} updated successfully` + message: `Skipping ${containerName} - cannot update Dockhand itself` }); - successCount++; + skippedCount++; + continue; + } - // Clear pending update indicator from database - if (envIdNum) { - await removePendingContainerUpdate(envIdNum, containerId).catch(() => { - // Ignore errors - record may not exist - }); - } + // Skip digest-pinned images - they are explicitly locked to a specific version + if (isDigestBasedImage(imageName)) { + sendData({ + type: 'progress', + containerId, + containerName, + step: 'skipped', + current: i + 1, + total: containerIds.length, + success: true, + message: `Skipping ${containerName} - image pinned to specific digest` + }); + skippedCount++; + continue; + } - } catch (error: any) { - safeEnqueue({ + // Step 1: Pull latest image + sendData({ + type: 'progress', + containerId, + containerName, + step: 'pulling', + current: i + 1, + total: containerIds.length, + message: `Pulling ${imageName}...` + }); + + try { + await pullImage(imageName, (data: any) => { + if (data.status) { + sendData({ + type: 'pull_log', + containerId, + containerName, + pullStatus: data.status, + pullId: data.id, + pullProgress: data.progress + }); + } + }, envIdNum); + } catch (pullError: any) { + sendData({ type: 'progress', containerId, containerName, @@ -520,53 +221,310 @@ export const POST: RequestHandler = async (event) => { current: i + 1, total: containerIds.length, success: false, - error: error.message + error: `Pull failed: ${pullError.message}` }); failCount++; + continue; } - } - // Send complete event - safeEnqueue({ - type: 'complete', - summary: { + // SAFE-PULL FLOW with vulnerability scanning + if (shouldScan && !isDigestBasedImage(imageName)) { + const tempTag = getTempImageTag(imageName); + + // Get new image ID + const newImageId = await getImageIdByTag(imageName, envIdNum); + if (!newImageId) { + sendData({ + type: 'progress', + containerId, + containerName, + step: 'failed', + current: i + 1, + total: containerIds.length, + success: false, + error: 'Failed to get new image ID after pull' + }); + failCount++; + continue; + } + + // Restore original tag to old image (safety) + const [oldRepo, oldTag] = parseImageNameAndTag(imageName); + try { + await tagImage(currentImageId, oldRepo, oldTag, envIdNum); + } catch { + // Ignore - old image might have been removed + } + + // Tag new image with temp suffix + const [tempRepo, tempTagName] = parseImageNameAndTag(tempTag); + await tagImage(newImageId, tempRepo, tempTagName, envIdNum); + + // Step 2: Scan temp image + sendData({ + type: 'scan_start', + containerId, + containerName, + step: 'scanning', + current: i + 1, + total: containerIds.length, + message: `Scanning ${imageName} for vulnerabilities...` + }); + + let scanBlocked = false; + let blockReason = ''; + let finalScanResult: ScanResult | undefined; + let individualScannerResults: ScannerResult[] = []; + + try { + const scanResults = await scanImage(tempTag, envIdNum, (progress) => { + if (progress.output || progress.message) { + sendData({ + type: 'scan_log', + containerId, + containerName, + scanner: progress.scanner, + message: progress.output || progress.message + }); + } + }); + + if (scanResults.length > 0) { + const scanSummary = combineScanSummaries(scanResults); + finalScanResult = { + critical: scanSummary.critical, + high: scanSummary.high, + medium: scanSummary.medium, + low: scanSummary.low, + negligible: scanSummary.negligible, + unknown: scanSummary.unknown + }; + + // Build individual scanner results + individualScannerResults = scanResults.map(result => ({ + scanner: result.scanner as 'grype' | 'trivy', + critical: result.summary.critical, + high: result.summary.high, + medium: result.summary.medium, + low: result.summary.low, + negligible: result.summary.negligible, + unknown: result.summary.unknown + })); + + // Save scan results + for (const result of scanResults) { + try { + await saveVulnerabilityScan({ + environmentId: envIdNum, + imageId: newImageId, + imageName: result.imageName, + scanner: result.scanner, + scannedAt: result.scannedAt, + scanDuration: result.scanDuration, + criticalCount: result.summary.critical, + highCount: result.summary.high, + mediumCount: result.summary.medium, + lowCount: result.summary.low, + negligibleCount: result.summary.negligible, + unknownCount: result.summary.unknown, + vulnerabilities: result.vulnerabilities, + error: result.error ?? null + }); + } catch { /* ignore save errors */ } + } + + // Check if blocked + const { blocked, reason } = shouldBlockUpdate(vulnerabilityCriteria, scanSummary, undefined); + if (blocked) { + scanBlocked = true; + blockReason = reason; + } + } + + // Collect vulnerabilities from all scanners (cap at 100) + const vulnerabilities = scanResults + .flatMap(r => r.vulnerabilities || []) + .slice(0, 100) + .map(v => ({ + id: v.id, + severity: v.severity, + package: v.package, + version: v.version, + fixedVersion: v.fixedVersion, + link: v.link, + scanner: v.scanner + })); + + sendData({ + type: 'scan_complete', + containerId, + containerName, + scanResult: finalScanResult, + scannerResults: individualScannerResults.length > 0 ? individualScannerResults : undefined, + vulnerabilities: vulnerabilities.length > 0 ? vulnerabilities : undefined, + message: finalScanResult + ? `Scan complete: ${finalScanResult.critical} critical, ${finalScanResult.high} high, ${finalScanResult.medium} medium, ${finalScanResult.low} low` + : 'Scan complete: no vulnerabilities found' + }); + + } catch (scanErr: any) { + sendData({ + type: 'progress', + containerId, + containerName, + step: 'failed', + current: i + 1, + total: containerIds.length, + success: false, + error: `Scan failed: ${scanErr.message}` + }); + + // Clean up temp image on scan failure + try { + await removeTempImage(newImageId, envIdNum); + } catch { /* ignore cleanup errors */ } + + failCount++; + continue; + } + + if (scanBlocked) { + // BLOCKED - Remove temp image and skip this container + sendData({ + type: 'blocked', + containerId, + containerName, + step: 'blocked', + current: i + 1, + total: containerIds.length, + success: false, + scanResult: finalScanResult, + scannerResults: individualScannerResults.length > 0 ? individualScannerResults : undefined, + blockReason, + message: `Update blocked: ${blockReason}` + }); + + try { + await removeTempImage(newImageId, envIdNum); + } catch { /* ignore cleanup errors */ } + + blockedCount++; + continue; + } + + // APPROVED - Re-tag to original + await tagImage(newImageId, oldRepo, oldTag, envIdNum); + try { + await removeTempImage(tempTag, envIdNum); + } catch { /* ignore cleanup errors */ } + } + + // Progress logging function for shared functions + const logProgress = (message: string) => { + sendData({ + type: 'progress', + containerId, + containerName, + step: 'creating', + current: i + 1, + total: containerIds.length, + message + }); + }; + + let newContainerId = containerId; + + sendData({ + type: 'progress', + containerId, + containerName, + step: 'creating', + current: i + 1, total: containerIds.length, - success: successCount, - failed: failCount, - blocked: blockedCount, - skipped: skippedCount - }, - message: skippedCount > 0 || blockedCount > 0 - ? `Updated ${successCount} of ${containerIds.length} containers${blockedCount > 0 ? ` (${blockedCount} blocked)` : ''}${skippedCount > 0 ? ` (${skippedCount} skipped)` : ''}` - : `Updated ${successCount} of ${containerIds.length} containers` - }); + message: `Recreating ${containerName}...` + }); - if (keepaliveInterval) { - clearInterval(keepaliveInterval); - } - if (!controllerClosed) { - try { - controller.close(); - controllerClosed = true; - } catch { - // Controller already closed - ignore - controllerClosed = true; + const recreateResult = await recreateContainer(containerName, envIdNum, logProgress, imageName); + if (recreateResult.success) { + const updatedContainers = await listContainers(true, envIdNum); + const updatedContainer = updatedContainers.find(c => c.name === containerName); + if (updatedContainer) { + newContainerId = updatedContainer.id; + } } - } - }, - cancel() { - controllerClosed = true; - if (keepaliveInterval) { - clearInterval(keepaliveInterval); + + if (!recreateResult.success) { + sendData({ + type: 'progress', + containerId, + containerName, + step: 'failed', + current: i + 1, + total: containerIds.length, + success: false, + error: recreateResult.error || 'Container recreation failed' + }); + failCount++; + continue; + } + + // Audit log + await auditContainer(event, 'update', newContainerId, containerName, envIdNum, { batchUpdate: true }); + + // Done with this container - use original containerId for UI consistency + sendData({ + type: 'progress', + containerId, + containerName, + step: 'done', + current: i + 1, + total: containerIds.length, + success: true, + message: `${containerName} updated successfully` + }); + successCount++; + + // Clear pending update indicator from database + if (envIdNum) { + await removePendingContainerUpdate(envIdNum, containerId).catch(() => { + // Ignore errors - record may not exist + }); + } + + } catch (error: any) { + sendData({ + type: 'progress', + containerId, + containerName, + step: 'failed', + current: i + 1, + total: containerIds.length, + success: false, + error: error.message + }); + failCount++; } } + + // Send complete event + const completeData: UpdateProgress = { + type: 'complete', + summary: { + total: containerIds.length, + success: successCount, + failed: failCount, + blocked: blockedCount, + skipped: skippedCount + }, + message: skippedCount > 0 || blockedCount > 0 + ? `Updated ${successCount} of ${containerIds.length} containers${blockedCount > 0 ? ` (${blockedCount} blocked)` : ''}${skippedCount > 0 ? ` (${skippedCount} skipped)` : ''}` + : `Updated ${successCount} of ${containerIds.length} containers` + }; + sendData(completeData); + completeJob(job, completeData); + })().catch((err) => { + failJob(job, err instanceof Error ? err.message : String(err)); }); - return new Response(stream, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive' - } - }); + return json({ jobId: job.id }); }; diff --git a/src/routes/api/containers/batch-update/+server.ts b/src/routes/api/containers/batch-update/+server.ts index 9711714..4cad151 100644 --- a/src/routes/api/containers/batch-update/+server.ts +++ b/src/routes/api/containers/batch-update/+server.ts @@ -75,11 +75,10 @@ export const POST: RequestHandler = async (event) => { continue; } - let updateSuccess = false; let newContainerId = containerId; - updateSuccess = await recreateContainer(containerName, envIdNum); - if (updateSuccess) { + const recreateResult = await recreateContainer(containerName, envIdNum); + if (recreateResult.success) { const updatedContainers = await listContainers(true, envIdNum); const updatedContainer = updatedContainers.find(c => c.name === containerName); if (updatedContainer) { @@ -87,12 +86,12 @@ export const POST: RequestHandler = async (event) => { } } - if (!updateSuccess) { + if (!recreateResult.success) { results.push({ containerId, containerName, success: false, - error: 'Container recreation failed' + error: recreateResult.error || 'Container recreation failed' }); continue; } diff --git a/src/routes/api/containers/stats/+server.ts b/src/routes/api/containers/stats/+server.ts index 22ed2dd..4429a5d 100644 --- a/src/routes/api/containers/stats/+server.ts +++ b/src/routes/api/containers/stats/+server.ts @@ -104,7 +104,7 @@ export const GET: RequestHandler = async ({ url, cookies }) => { // Get all running containers with timeout const containers = await withTimeout( listContainers(true, envIdNum), - 5000, // 5 second timeout + 10000, // 10 second timeout [] ); const runningContainers = containers.filter(c => c.state === 'running'); @@ -127,7 +127,7 @@ export const GET: RequestHandler = async ({ url, cookies }) => { try { const stats = await withTimeout( getContainerStats(container.id, envIdNum) as Promise, - 3000, // 3 second timeout per container + 8000, // 8 second timeout per container (TLS proxy + Docker CPU sampling needs ~2s) null ); diff --git a/src/routes/api/containers/stats/stream/+server.ts b/src/routes/api/containers/stats/stream/+server.ts new file mode 100644 index 0000000..8922c67 --- /dev/null +++ b/src/routes/api/containers/stats/stream/+server.ts @@ -0,0 +1,182 @@ +import type { RequestHandler } from './$types'; +import { listContainers, getContainerStats, EnvironmentNotFoundError } from '$lib/server/docker'; +import { authorize } from '$lib/server/authorize'; +import { hasEnvironments } from '$lib/server/db'; +import type { ContainerStats } from '$lib/types'; + +function calculateCpuPercent(stats: any): number { + const cpuDelta = stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage; + const systemDelta = stats.cpu_stats.system_cpu_usage - stats.precpu_stats.system_cpu_usage; + const cpuCount = stats.cpu_stats.online_cpus || stats.cpu_stats.cpu_usage.percpu_usage?.length || 1; + + if (systemDelta > 0 && cpuDelta > 0) { + return (cpuDelta / systemDelta) * cpuCount * 100; + } + return 0; +} + +function calculateNetworkIO(stats: any): { rx: number; tx: number } { + let rx = 0; + let tx = 0; + + if (stats.networks) { + for (const iface of Object.values(stats.networks) as any[]) { + rx += iface.rx_bytes || 0; + tx += iface.tx_bytes || 0; + } + } + + return { rx, tx }; +} + +function calculateBlockIO(stats: any): { read: number; write: number } { + let read = 0; + let write = 0; + + const ioStats = stats.blkio_stats?.io_service_bytes_recursive; + if (Array.isArray(ioStats)) { + for (const entry of ioStats) { + if (entry.op === 'read' || entry.op === 'Read') { + read += entry.value || 0; + } else if (entry.op === 'write' || entry.op === 'Write') { + write += entry.value || 0; + } + } + } + + return { read, write }; +} + +function calculateMemoryUsage(memoryStats: any): { usage: number; raw: number; cache: number } { + const raw = memoryStats?.usage || 0; + const stats = memoryStats?.stats || {}; + const cache = stats.inactive_file ?? stats.total_inactive_file ?? 0; + const usage = (cache > 0 && cache < raw) ? raw - cache : raw; + return { usage, raw, cache }; +} + +function withTimeout(promise: Promise, ms: number, fallback: T): Promise { + let timeoutId: ReturnType | null = null; + const timeoutPromise = new Promise((resolve) => { + timeoutId = setTimeout(() => resolve(fallback), ms); + }); + return Promise.race([promise, timeoutPromise]).finally(() => { + if (timeoutId !== null) clearTimeout(timeoutId); + }); +} + +export const GET: RequestHandler = async ({ url, cookies }) => { + const auth = await authorize(cookies); + + const envId = url.searchParams.get('env'); + const envIdNum = envId ? parseInt(envId) : undefined; + + if (auth.authEnabled && !await auth.can('containers', 'view', envIdNum)) { + return new Response(JSON.stringify({ error: 'Permission denied' }), { + status: 403, + headers: { 'Content-Type': 'application/json' } + }); + } + + if (!await hasEnvironments() || !envIdNum) { + return new Response('event: done\ndata: {}\n\n', { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'X-Accel-Buffering': 'no' + } + }); + } + + let controllerClosed = false; + const stream = new ReadableStream({ + async start(controller) { + const encoder = new TextEncoder(); + + const safeEnqueue = (data: string) => { + if (!controllerClosed) { + try { + controller.enqueue(encoder.encode(data)); + } catch { + controllerClosed = true; + } + } + }; + + try { + const containers = await withTimeout( + listContainers(true, envIdNum), + 10000, + [] + ); + const runningContainers = containers.filter(c => c.state === 'running'); + + const statsPromises = runningContainers.map(async (container) => { + try { + const stats = await withTimeout( + getContainerStats(container.id, envIdNum) as Promise, + 8000, + null + ); + + if (!stats) return; + + const cpuPercent = calculateCpuPercent(stats); + const memory = calculateMemoryUsage(stats.memory_stats); + const memoryLimit = stats.memory_stats?.limit || 1; + const memoryPercent = (memory.usage / memoryLimit) * 100; + const networkIO = calculateNetworkIO(stats); + const blockIO = calculateBlockIO(stats); + + const stat: ContainerStats = { + id: container.id, + name: container.name, + cpuPercent: Math.round(cpuPercent * 100) / 100, + memoryUsage: memory.usage, + memoryRaw: memory.raw, + memoryCache: memory.cache, + memoryLimit, + memoryPercent: Math.round(memoryPercent * 100) / 100, + networkRx: networkIO.rx, + networkTx: networkIO.tx, + blockRead: blockIO.read, + blockWrite: blockIO.write + }; + + safeEnqueue(`event: stat\ndata: ${JSON.stringify(stat)}\n\n`); + } catch { + // Skip failed containers silently + } + }); + + await Promise.all(statsPromises); + } catch (error: any) { + if (error instanceof EnvironmentNotFoundError) { + safeEnqueue(`event: error\ndata: ${JSON.stringify({ error: 'Environment not found' })}\n\n`); + } + } + + if (!controllerClosed) { + safeEnqueue(`event: done\ndata: {}\n\n`); + try { + controller.close(); + } catch { + // Already closed + } + } + }, + cancel() { + controllerClosed = true; + } + }); + + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'X-Accel-Buffering': 'no' + } + }); +}; diff --git a/src/routes/api/dashboard/stats/stream/+server.ts b/src/routes/api/dashboard/stats/stream/+server.ts index 69cafbd..c31e047 100644 --- a/src/routes/api/dashboard/stats/stream/+server.ts +++ b/src/routes/api/dashboard/stats/stream/+server.ts @@ -3,6 +3,7 @@ import { getEnvironments, getLatestHostMetrics, getHostMetrics, + getMetricsCollectionInterval, getContainerEventStats, getContainerEvents, getEnvSetting, @@ -14,13 +15,17 @@ import { listImages, listNetworks, getContainerStats, - getDiskUsage + getDiskUsage, + dockerPing, + DockerConnectionError } from '$lib/server/docker'; import { listComposeStacks } from '$lib/server/stacks'; import { authorize } from '$lib/server/authorize'; +import { prefersJSON, sseToJSON } from '$lib/server/sse'; import type { EnvironmentStats } from '../+server'; import { parseLabels } from '$lib/utils/label-colors'; + // Skip disk usage collection (Synology NAS performance fix) const SKIP_DF_COLLECTION = process.env.SKIP_DF_COLLECTION === 'true' || process.env.SKIP_DF_COLLECTION === '1'; @@ -68,6 +73,9 @@ setInterval(() => { } }, 10 * 60 * 1000); // Every 10 minutes +// Register cache reporter for memory monitoring + + async function getCachedDiskUsage(envId: number): Promise { const cached = diskUsageCache.get(envId); const now = Date.now(); @@ -125,10 +133,14 @@ function calculateMemoryUsage(memoryStats: any): number { return usage; } +// Target time window for metrics history charts (15 minutes) +const METRICS_HISTORY_WINDOW_MS = 15 * 60 * 1000; + // Progressive stats loading - returns stats object and emits partial updates via callback async function getEnvironmentStatsProgressive( env: any, - onPartialUpdate: (stats: Partial & { id: number }) => void + onPartialUpdate: (stats: Partial & { id: number }) => void, + metricsPointCount: number ): Promise { const envStats: EnvironmentStats = { id: env.id, @@ -187,7 +199,7 @@ async function getEnvironmentStatsProgressive( getLatestHostMetrics(env.id), getContainerEventStats(env.id), getContainerEvents({ environmentId: env.id, limit: 10 }), - getHostMetrics(30, env.id), + getHostMetrics(metricsPointCount, env.id), getPendingContainerUpdates(env.id) ]); @@ -237,6 +249,20 @@ async function getEnvironmentStatsProgressive( loading: { ...envStats.loading } }); + // Quick reachability check — if ping fails, skip all expensive Docker API calls + if (!await dockerPing(env.id)) { + envStats.online = false; + envStats.error = 'Environment offline'; + envStats.loading = undefined; + onPartialUpdate({ + id: env.id, + online: false, + error: 'Environment offline', + loading: undefined + }); + return envStats; + } + // Helper to get valid size const getValidSize = (size: number | undefined | null): number => { return size && size > 0 ? size : 0; @@ -511,7 +537,7 @@ async function getEnvironmentStatsProgressive( return envStats; } -export const GET: RequestHandler = async ({ cookies }) => { +export const GET: RequestHandler = async ({ request, cookies }) => { const auth = await authorize(cookies); if (auth.authEnabled && !await auth.can('environments', 'view')) { return new Response(JSON.stringify({ error: 'Permission denied' }), { @@ -535,6 +561,7 @@ export const GET: RequestHandler = async ({ cookies }) => { let controllerClosed = false; const stream = new ReadableStream({ async start(controller) { + const encoder = new TextEncoder(); // Safe enqueue that checks if controller is still open @@ -573,17 +600,23 @@ export const GET: RequestHandler = async ({ cookies }) => { })); safeEnqueue(`event: environments\ndata: ${JSON.stringify(envList)}\n\n`); + // Calculate metrics point count based on configured interval + const metricsIntervalMs = await getMetricsCollectionInterval(); + const metricsPointCount = Math.ceil(METRICS_HISTORY_WINDOW_MS / metricsIntervalMs); + // Fetch stats for each environment with progressive updates const promises = environments.map(async (env) => { try { await getEnvironmentStatsProgressive(env, (partialStats) => { // Send partial update as it arrives safeEnqueue(`event: partial\ndata: ${JSON.stringify(partialStats)}\n\n`); - }); + }, metricsPointCount); // Send final complete stats event for this environment safeEnqueue(`event: complete\ndata: ${JSON.stringify({ id: env.id })}\n\n`); } catch (error) { - console.error(`Failed to get stats for ${env.name}:`, error); + if (!(error instanceof DockerConnectionError)) { + console.error(`Failed to get stats for ${env.name}:`, error); + } // Convert technical error to user-friendly message const errorStr = String(error); let friendlyError = 'Connection error'; @@ -611,19 +644,25 @@ export const GET: RequestHandler = async ({ cookies }) => { } catch { // Already closed } + } }, cancel() { // Called when the client disconnects controllerClosed = true; + } }); - return new Response(stream, { + const sseResponse = new Response(stream, { headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive' + 'Connection': 'keep-alive', + 'X-Accel-Buffering': 'no' } }); + + if (prefersJSON(request)) return sseToJSON(sseResponse); + return sseResponse; }; diff --git a/src/routes/api/debug/memory/+server.ts b/src/routes/api/debug/memory/+server.ts new file mode 100644 index 0000000..765968e --- /dev/null +++ b/src/routes/api/debug/memory/+server.ts @@ -0,0 +1,121 @@ +/** + * Memory Debug Endpoint + * + * Returns Node.js memory stats for monitoring. + * Only available when MEMORY_MONITOR=true environment variable is set. + * + * GET /api/debug/memory - Memory stats (with optional ?gc=true to force GC first) + * GET /api/debug/memory?gc=true - Force garbage collection before reporting + */ + +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import v8 from 'node:v8'; +import os from 'node:os'; +import { getRssStats, dumpHeapSnapshot, listHeapSnapshots } from '$lib/server/rss-tracker'; + +// Track startup time and initial RSS for growth rate calculation +const startupTime = Date.now(); +const startupRss = process.memoryUsage().rss; + +export const GET: RequestHandler = async ({ url }) => { + if (process.env.MEMORY_MONITOR !== 'true') { + return json({ error: 'Memory monitor not enabled. Set MEMORY_MONITOR=true.' }, { status: 403 }); + } + + // Trigger manual heap snapshot + if (url.searchParams.has('snapshot')) { + const filename = dumpHeapSnapshot(); + return json({ + snapshot: filename ? { filename, message: 'Heap snapshot saved' } : { error: 'Failed to save snapshot' } + }); + } + + // List saved snapshots + if (url.searchParams.has('snapshots')) { + return json({ snapshots: listHeapSnapshots() }); + } + + // Force GC if requested and available + const forceGc = url.searchParams.get('gc') === 'true'; + if (forceGc && typeof globalThis.gc === 'function') { + globalThis.gc(); + } + + const mem = process.memoryUsage(); + const heap = v8.getHeapStatistics(); + const uptimeMs = Date.now() - startupTime; + const uptimeHours = uptimeMs / (1000 * 60 * 60); + const rssGrowth = mem.rss - startupRss; + const rssGrowthPerHour = uptimeHours > 0.01 ? rssGrowth / uptimeHours : 0; + + return json({ + timestamp: new Date().toISOString(), + uptime: { + ms: uptimeMs, + hours: Math.round(uptimeHours * 100) / 100, + human: formatUptime(uptimeMs), + }, + gcForced: forceGc && typeof globalThis.gc === 'function', + gcAvailable: typeof globalThis.gc === 'function', + process: { + rss: formatBytes(mem.rss), + heapTotal: formatBytes(mem.heapTotal), + heapUsed: formatBytes(mem.heapUsed), + external: formatBytes(mem.external), + arrayBuffers: formatBytes(mem.arrayBuffers), + rssRaw: mem.rss, + heapTotalRaw: mem.heapTotal, + heapUsedRaw: mem.heapUsed, + externalRaw: mem.external, + arrayBuffersRaw: mem.arrayBuffers, + }, + growth: { + rssSinceStartup: formatBytes(rssGrowth), + rssPerHour: formatBytes(Math.round(rssGrowthPerHour)), + startupRss: formatBytes(startupRss), + }, + v8Heap: { + totalHeapSize: formatBytes(heap.total_heap_size), + usedHeapSize: formatBytes(heap.used_heap_size), + heapSizeLimit: formatBytes(heap.heap_size_limit), + totalPhysicalSize: formatBytes(heap.total_physical_size), + totalAvailableSize: formatBytes(heap.total_available_size), + mallocedMemory: formatBytes(heap.malloced_memory), + peakMallocedMemory: formatBytes(heap.peak_malloced_memory), + externalMemory: formatBytes(heap.external_memory), + numberOfNativeContexts: heap.number_of_native_contexts, + numberOfDetachedContexts: heap.number_of_detached_contexts, + }, + system: { + totalMemory: formatBytes(os.totalmem()), + freeMemory: formatBytes(os.freemem()), + cpus: os.cpus().length, + platform: os.platform(), + arch: os.arch(), + nodeVersion: process.version, + }, + rssTracker: getRssStats(), + }); +}; + +function formatBytes(bytes: number): string { + if (bytes === 0) return '0 B'; + const sign = bytes < 0 ? '-' : ''; + const abs = Math.abs(bytes); + if (abs < 1024) return `${sign}${abs} B`; + if (abs < 1024 * 1024) return `${sign}${(abs / 1024).toFixed(1)} KB`; + if (abs < 1024 * 1024 * 1024) return `${sign}${(abs / (1024 * 1024)).toFixed(1)} MB`; + return `${sign}${(abs / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} + +function formatUptime(ms: number): string { + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + if (days > 0) return `${days}d ${hours % 24}h ${minutes % 60}m`; + if (hours > 0) return `${hours}h ${minutes % 60}m`; + if (minutes > 0) return `${minutes}m ${seconds % 60}s`; + return `${seconds}s`; +} diff --git a/src/routes/api/environments/+server.ts b/src/routes/api/environments/+server.ts index 393a934..b4f6941 100644 --- a/src/routes/api/environments/+server.ts +++ b/src/routes/api/environments/+server.ts @@ -113,7 +113,7 @@ export const POST: RequestHandler = async (event) => { await setEnvironmentPublicIp(env.id, data.publicIp); } - // Notify subprocesses to pick up the new environment + // Notify event collectors to pick up the new environment refreshSubprocessEnvironments(); // Auto-assign Admin role to creator (Enterprise only) diff --git a/src/routes/api/environments/[id]/+server.ts b/src/routes/api/environments/[id]/+server.ts index 97d77d7..a939acc 100644 --- a/src/routes/api/environments/[id]/+server.ts +++ b/src/routes/api/environments/[id]/+server.ts @@ -91,7 +91,7 @@ export const PUT: RequestHandler = async (event) => { return json({ error: 'Environment not found' }, { status: 404 }); } - // Notify subprocesses if collectActivity or collectMetrics setting changed + // Notify event collectors if collectActivity or collectMetrics setting changed if (data.collectActivity !== undefined || data.collectMetrics !== undefined) { refreshSubprocessEnvironments(); } @@ -178,7 +178,7 @@ export const DELETE: RequestHandler = async (event) => { await deleteImagePruneSettings(id); unregisterSchedule(id, 'image_prune'); - // Notify subprocesses to stop collecting from deleted environment + // Notify event collectors to stop collecting from deleted environment refreshSubprocessEnvironments(); // Audit log diff --git a/src/routes/api/environments/[id]/test/+server.ts b/src/routes/api/environments/[id]/test/+server.ts index 18579df..374b45b 100644 --- a/src/routes/api/environments/[id]/test/+server.ts +++ b/src/routes/api/environments/[id]/test/+server.ts @@ -71,7 +71,7 @@ export const POST: RequestHandler = async ({ params }) => { } // For Hawser Standard mode, fetch Docker info and Hawser info in parallel - // (sequential calls can fail due to Bun TLS connection reuse issues) + // (parallel calls are more efficient and avoid sequential connection issues) let info: any; let hawserInfo = null; if (env.connectionType === 'hawser-standard') { diff --git a/src/routes/api/environments/[id]/timezone/+server.ts b/src/routes/api/environments/[id]/timezone/+server.ts index e5b81ce..1ae177a 100644 --- a/src/routes/api/environments/[id]/timezone/+server.ts +++ b/src/routes/api/environments/[id]/timezone/+server.ts @@ -8,7 +8,7 @@ import { } from '$lib/server/db'; import { refreshSchedulesForEnvironment } from '$lib/server/scheduler'; -/** Map of modern IANA timezone names to their canonical equivalents recognized by Bun/ICU */ +/** Map of modern IANA timezone names to their canonical equivalents recognized by ICU */ const TIMEZONE_ALIASES: Record = { 'Europe/Kyiv': 'Europe/Kiev', 'Asia/Ho_Chi_Minh': 'Asia/Saigon', diff --git a/src/routes/api/environments/test/+server.ts b/src/routes/api/environments/test/+server.ts index d5300ab..3027680 100644 --- a/src/routes/api/environments/test/+server.ts +++ b/src/routes/api/environments/test/+server.ts @@ -1,4 +1,6 @@ import { json } from '@sveltejs/kit'; +import { unixSocketRequest, httpsAgentRequest } from '$lib/server/docker'; +import type { DockerClientConfig } from '$lib/server/docker'; import type { RequestHandler } from './$types'; interface TestConnectionRequest { @@ -22,29 +24,19 @@ function cleanPem(pem: string): string { .join('\n'); } -function buildTlsOptions(config: TestConnectionRequest): Record | undefined { +function buildDockerClientConfig(config: TestConnectionRequest): DockerClientConfig | null { const protocol = config.protocol || 'http'; - if (protocol !== 'https') return undefined; + if (protocol !== 'https') return null; - const tls: Record = { - sessionTimeout: 0, - servername: config.host + return { + type: 'https', + host: config.host || 'localhost', + port: config.port || 2376, + ca: config.tlsCa ? cleanPem(config.tlsCa) || undefined : undefined, + cert: config.tlsCert ? cleanPem(config.tlsCert) || undefined : undefined, + key: config.tlsKey ? cleanPem(config.tlsKey) || undefined : undefined, + skipVerify: config.tlsSkipVerify || false }; - if (config.tlsSkipVerify) { - tls.rejectUnauthorized = false; - } else { - tls.rejectUnauthorized = true; - if (config.tlsCa) { - tls.ca = [cleanPem(config.tlsCa)]; - } - } - if (config.tlsCert) { - tls.cert = [cleanPem(config.tlsCert)]; - } - if (config.tlsKey) { - tls.key = cleanPem(config.tlsKey); - } - return tls; } /** @@ -59,11 +51,7 @@ export const POST: RequestHandler = async ({ request }) => { if (config.connectionType === 'socket') { const socketPath = config.socketPath || '/var/run/docker.sock'; - response = await fetch('http://localhost/info', { - // @ts-ignore - Bun supports unix socket - unix: socketPath, - signal: AbortSignal.timeout(10000) - }); + response = await unixSocketRequest(socketPath, '/info'); } else if (config.connectionType === 'hawser-edge') { // Edge mode - cannot test directly, agent connects to us return json({ @@ -83,7 +71,6 @@ export const POST: RequestHandler = async ({ request }) => { return json({ success: false, error: 'Host is required' }, { status: 400 }); } - const url = `${protocol}://${host}:${port}/info`; const headers: Record = { 'Content-Type': 'application/json' }; @@ -92,19 +79,17 @@ export const POST: RequestHandler = async ({ request }) => { headers['X-Hawser-Token'] = config.hawserToken; } - const fetchOptions: any = { - headers, - signal: AbortSignal.timeout(10000), - keepalive: false - }; - - const tls = buildTlsOptions(config); - if (tls) { - fetchOptions.tls = tls; - if (process.env.DEBUG_TLS) fetchOptions.verbose = true; + const tlsConfig = buildDockerClientConfig(config); + if (tlsConfig) { + response = await httpsAgentRequest(tlsConfig, '/info', {}, false, headers); + } else { + const url = `http://${host}:${port}/info`; + response = await fetch(url, { + headers, + signal: AbortSignal.timeout(10000), + keepalive: false + }); } - - response = await fetch(url, fetchOptions); } if (!response.ok) { @@ -123,21 +108,19 @@ export const POST: RequestHandler = async ({ request }) => { if (config.hawserToken) { hawserHeaders['X-Hawser-Token'] = config.hawserToken; } - const hawserUrl = `${protocol}://${config.host}:${config.port || 2375}/_hawser/info`; - const fetchOptions: any = { - headers: hawserHeaders, - signal: AbortSignal.timeout(5000), - keepalive: false - }; - - const tls = buildTlsOptions(config); - if (tls) { - fetchOptions.tls = tls; - if (process.env.DEBUG_TLS) fetchOptions.verbose = true; + let hawserResp: Response; + const tlsConfig = buildDockerClientConfig(config); + if (tlsConfig) { + hawserResp = await httpsAgentRequest(tlsConfig, '/_hawser/info', {}, false, hawserHeaders); + } else { + const hawserUrl = `http://${config.host}:${config.port || 2375}/_hawser/info`; + hawserResp = await fetch(hawserUrl, { + headers: hawserHeaders, + signal: AbortSignal.timeout(5000), + keepalive: false + }); } - - const hawserResp = await fetch(hawserUrl, fetchOptions); if (hawserResp.ok) { hawserInfo = await hawserResp.json(); } @@ -182,6 +165,8 @@ export const POST: RequestHandler = async ({ request }) => { message = 'Connection failed - check host and port'; } else if (rawMessage.includes('self signed certificate') || rawMessage.includes('UNABLE_TO_VERIFY_LEAF_SIGNATURE')) { message = 'TLS certificate error - provide CA certificate for self-signed certs'; + } else if (rawMessage.includes('CERT_ALTNAME_INVALID') || rawMessage.includes('ERR_TLS_CERT_ALTNAME_INVALID')) { + message = 'Certificate hostname mismatch - your certificate\'s Subject Alternative Name (SAN) doesn\'t match the host. Regenerate with: -addext "subjectAltName=DNS:hostname,IP:x.x.x.x"'; } else if (rawMessage.includes('certificate') || rawMessage.includes('SSL') || rawMessage.includes('TLS')) { message = 'TLS/SSL error - check certificate configuration'; } diff --git a/src/routes/api/git/stacks/+server.ts b/src/routes/api/git/stacks/+server.ts index 8719d98..9429d68 100644 --- a/src/routes/api/git/stacks/+server.ts +++ b/src/routes/api/git/stacks/+server.ts @@ -14,6 +14,7 @@ import { authorize } from '$lib/server/authorize'; import { registerSchedule } from '$lib/server/scheduler'; import { secureRandomBytes } from '$lib/server/crypto-fallback'; import { auditGitStack } from '$lib/server/audit'; +import { createJobResponse } from '$lib/server/sse'; // Stack name validation: must start with alphanumeric, can contain alphanumeric, hyphens, underscores const STACK_NAME_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/; @@ -156,20 +157,33 @@ export const POST: RequestHandler = async (event) => { } } - // If deployNow is set, deploy immediately + // If deployNow is set, deploy immediately via SSE to keep connection alive if (data.deployNow) { - const deployResult = await deployGitStack(gitStack.id); - await auditGitStack(event, 'deploy', gitStack.id, gitStack.stackName, gitStack.environmentId); - return json({ - ...gitStack, - deployResult: deployResult - }); + return createJobResponse(async (send) => { + try { + const deployResult = await deployGitStack(gitStack.id); + await auditGitStack(event, 'deploy', gitStack.id, gitStack.stackName, gitStack.environmentId); + send('result', { + ...gitStack, + deployResult: deployResult + }); + } catch (error) { + console.error('Failed to deploy git stack:', error); + send('result', { + ...gitStack, + deployResult: { success: false, error: 'Failed to deploy git stack' } + }); + } + }, request); } return json(gitStack); } catch (error: any) { console.error('Failed to create git stack:', error); if (error.message?.includes('UNIQUE constraint failed')) { + if (error.message?.includes('stack_environment_variables')) { + return json({ error: 'Duplicate environment variable keys detected' }, { status: 400 }); + } return json({ error: 'A git stack with this name already exists for this environment' }, { status: 400 }); } return json({ error: 'Failed to create git stack' }, { status: 500 }); diff --git a/src/routes/api/git/stacks/[id]/+server.ts b/src/routes/api/git/stacks/[id]/+server.ts index b8a968c..8833d53 100644 --- a/src/routes/api/git/stacks/[id]/+server.ts +++ b/src/routes/api/git/stacks/[id]/+server.ts @@ -1,11 +1,12 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -import { getGitStack, updateGitStack, deleteGitStack, deleteStackSource, updateStackSourceName, updateStackEnvVarsName, setStackEnvVars, getStackEnvVars } from '$lib/server/db'; +import { getGitStack, updateGitStack, deleteGitStack, deleteStackSource, updateStackSourceName, updateStackEnvVarsName, setStackEnvVars, getStackEnvVars, deleteStackEnvVars } from '$lib/server/db'; import { deleteGitStackFiles, deployGitStack } from '$lib/server/git'; import { authorize } from '$lib/server/authorize'; import { registerSchedule, unregisterSchedule } from '$lib/server/scheduler'; import { auditGitStack } from '$lib/server/audit'; import { computeAuditDiff } from '$lib/utils/diff'; +import { createJobResponse } from '$lib/server/sse'; // Stack name validation: must start with alphanumeric, can contain alphanumeric, hyphens, underscores const STACK_NAME_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/; @@ -131,20 +132,33 @@ export const PUT: RequestHandler = async (event) => { await setStackEnvVars(stackName, envId, varsToSave as any); } - // If deployNow is set, deploy after saving + // If deployNow is set, deploy after saving via SSE to keep connection alive if (data.deployNow) { - const deployResult = await deployGitStack(id); - await auditGitStack(event, 'deploy', updated.id, updated.stackName, updated.environmentId); - return json({ - ...updated, - deployResult - }); + return createJobResponse(async (send) => { + try { + const deployResult = await deployGitStack(id); + await auditGitStack(event, 'deploy', updated.id, updated.stackName, updated.environmentId); + send('result', { + ...updated, + deployResult + }); + } catch (error) { + console.error('Failed to deploy git stack:', error); + send('result', { + ...updated, + deployResult: { success: false, error: 'Failed to deploy git stack' } + }); + } + }, request); } return json(updated); } catch (error: any) { console.error('Failed to update git stack:', error); if (error.message?.includes('UNIQUE constraint failed')) { + if (error.message?.includes('stack_environment_variables')) { + return json({ error: 'Duplicate environment variable keys detected' }, { status: 400 }); + } return json({ error: 'A git stack with this name already exists for this environment' }, { status: 400 }); } return json({ error: 'Failed to update git stack' }, { status: 500 }); @@ -176,6 +190,9 @@ export const DELETE: RequestHandler = async (event) => { // Delete the stack_sources record to free up the stack name await deleteStackSource(existing.stackName, existing.environmentId); + // Delete all env var overrides for this stack (all environments) + await deleteStackEnvVars(existing.stackName); + // Delete from database await deleteGitStack(id); diff --git a/src/routes/api/git/stacks/[id]/deploy-stream/+server.ts b/src/routes/api/git/stacks/[id]/deploy-stream/+server.ts index b2b8435..55eb6e5 100644 --- a/src/routes/api/git/stacks/[id]/deploy-stream/+server.ts +++ b/src/routes/api/git/stacks/[id]/deploy-stream/+server.ts @@ -3,8 +3,10 @@ import type { RequestHandler } from './$types'; import { getGitStack } from '$lib/server/db'; import { deployGitStackWithProgress } from '$lib/server/git'; import { authorize } from '$lib/server/authorize'; +import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs'; +import { prefersJSON, sseToJSON } from '$lib/server/sse'; -export const POST: RequestHandler = async ({ params, cookies }) => { +export const POST: RequestHandler = async ({ params, cookies, request }) => { const auth = await authorize(cookies); const id = parseInt(params.id); @@ -25,30 +27,43 @@ export const POST: RequestHandler = async ({ params, cookies }) => { }); } - // Create a readable stream for SSE - const stream = new ReadableStream({ - async start(controller) { - const encoder = new TextEncoder(); - - const sendEvent = (data: any) => { - controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`)); - }; - - try { - await deployGitStackWithProgress(id, sendEvent); - } catch (error: any) { - sendEvent({ status: 'error', error: error.message || 'Unknown error' }); - } finally { - controller.close(); + // Backward compat: API clients sending Accept: application/json get synchronous SSE result + if (prefersJSON(request)) { + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + async start(controller) { + const sendEvent = (data: unknown) => { + controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`)); + }; + try { + await deployGitStackWithProgress(id, sendEvent); + } catch (error: any) { + sendEvent({ status: 'error', error: error.message || 'Unknown error' }); + } finally { + controller.close(); + } } - } - }); + }); + const sseResponse = new Response(stream, { + headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' } + }); + return sseToJSON(sseResponse); + } - return new Response(stream, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive' - } - }); + // Job pattern: fire and forget, return jobId immediately + const job = createJob(); + + deployGitStackWithProgress(id, (data: unknown) => { + appendLine(job, { data }); + }) + .then(() => { + const lastLine = job.lines[job.lines.length - 1]; + const lastData = lastLine?.data as any; + completeJob(job, lastData ?? { status: 'complete' }); + }) + .catch((err: unknown) => { + failJob(job, err instanceof Error ? err.message : String(err)); + }); + + return json({ jobId: job.id }); }; diff --git a/src/routes/api/git/stacks/[id]/deploy/+server.ts b/src/routes/api/git/stacks/[id]/deploy/+server.ts index 09ed8fb..1cbbd50 100644 --- a/src/routes/api/git/stacks/[id]/deploy/+server.ts +++ b/src/routes/api/git/stacks/[id]/deploy/+server.ts @@ -4,6 +4,7 @@ import { getGitStack } from '$lib/server/db'; import { deployGitStack } from '$lib/server/git'; import { authorize } from '$lib/server/authorize'; import { auditGitStack } from '$lib/server/audit'; +import { createJobResponse } from '$lib/server/sse'; export const POST: RequestHandler = async (event) => { const { params, cookies } = event; @@ -21,12 +22,19 @@ export const POST: RequestHandler = async (event) => { return json({ error: 'Permission denied' }, { status: 403 }); } - const result = await deployGitStack(id); + return createJobResponse(async (send) => { + try { + const result = await deployGitStack(id); - // Audit log - await auditGitStack(event, 'deploy', id, gitStack.stackName, gitStack.environmentId); + // Audit log + await auditGitStack(event, 'deploy', id, gitStack.stackName, gitStack.environmentId); - return json(result); + send('result', result); + } catch (error) { + console.error('Failed to deploy git stack:', error); + send('result', { success: false, error: 'Failed to deploy git stack' }); + } + }, event.request); } catch (error) { console.error('Failed to deploy git stack:', error); return json({ error: 'Failed to deploy git stack' }, { status: 500 }); diff --git a/src/routes/api/hawser/connect/+server.ts b/src/routes/api/hawser/connect/+server.ts index 71f1da9..57d368d 100644 --- a/src/routes/api/hawser/connect/+server.ts +++ b/src/routes/api/hawser/connect/+server.ts @@ -2,7 +2,7 @@ * Hawser Edge WebSocket Connect Endpoint * * This endpoint handles WebSocket connections from Hawser agents running in Edge mode. - * In development: WebSocket is handled by Bun.serve in vite.config.ts on port 5174 + * In development: WebSocket is handled by ws.WebSocketServer in vite.config.ts on port 5174 * In production: WebSocket is handled by the server wrapper in server.ts * * The HTTP GET endpoint returns connection info for clients. @@ -28,7 +28,7 @@ export const GET: RequestHandler = async () => { hostname: conn.hostname, capabilities: conn.capabilities, connectedAt: conn.connectedAt.toISOString(), - lastHeartbeat: conn.lastHeartbeat.toISOString() + lastHeartbeat: new Date(conn.lastHeartbeat).toISOString() })); return json({ diff --git a/src/routes/api/images/+server.ts b/src/routes/api/images/+server.ts index 104f397..5198a13 100644 --- a/src/routes/api/images/+server.ts +++ b/src/routes/api/images/+server.ts @@ -1,5 +1,5 @@ import { json } from '@sveltejs/kit'; -import { listImages, EnvironmentNotFoundError } from '$lib/server/docker'; +import { listImages, EnvironmentNotFoundError, DockerConnectionError } from '$lib/server/docker'; import { authorize } from '$lib/server/authorize'; import { hasEnvironments } from '$lib/server/db'; import type { RequestHandler } from './$types'; @@ -32,7 +32,9 @@ export const GET: RequestHandler = async ({ url, cookies }) => { if (error instanceof EnvironmentNotFoundError) { return json({ error: 'Environment not found' }, { status: 404 }); } - console.error('Error listing images:', error); + if (!(error instanceof DockerConnectionError)) { + console.error('Error listing images:', error); + } // Return empty array instead of error to allow UI to load return json([]); } diff --git a/src/routes/api/images/pull/+server.ts b/src/routes/api/images/pull/+server.ts index 7da37d7..909b2db 100644 --- a/src/routes/api/images/pull/+server.ts +++ b/src/routes/api/images/pull/+server.ts @@ -1,11 +1,12 @@ import { json } from '@sveltejs/kit'; -import { pullImage } from '$lib/server/docker'; +import { pullImage, buildRegistryAuthHeader } from '$lib/server/docker'; import type { RequestHandler } from './$types'; import { getScannerSettings, scanImage } from '$lib/server/scanner'; import { saveVulnerabilityScan, getEnvironment } from '$lib/server/db'; import { authorize } from '$lib/server/authorize'; import { auditImage } from '$lib/server/audit'; import { sendEdgeStreamRequest, isEdgeConnected } from '$lib/server/hawser'; +import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs'; /** * Check if environment is edge mode @@ -73,49 +74,26 @@ export const POST: RequestHandler = async (event) => { // Check if this is an edge environment const edgeCheck = await isEdgeMode(envId); - const encoder = new TextEncoder(); - let controllerClosed = false; - let controller: ReadableStreamDefaultController; - let heartbeatInterval: ReturnType | null = null; - let cancelEdgeStream: (() => void) | null = null; + // Job pattern: create job, run in background, return jobId immediately + const job = createJob(); - const safeEnqueue = (data: string) => { - if (!controllerClosed) { - try { - controller.enqueue(encoder.encode(data)); - } catch { - controllerClosed = true; - } - } - }; - - const cleanup = () => { - if (heartbeatInterval) { - clearInterval(heartbeatInterval); - heartbeatInterval = null; - } - if (cancelEdgeStream) { - cancelEdgeStream(); - cancelEdgeStream = null; - } - controllerClosed = true; + const sendData = (data: unknown) => { + appendLine(job, { data }); }; /** * Handle scan-on-pull after image is pulled */ const handleScanOnPull = async () => { - // Skip if caller explicitly requested no scan (e.g., CreateContainerModal handles scanning separately) if (skipScanOnPull) return; const { scanner } = await getScannerSettings(envId); - // Scan if scanning is enabled (scanner !== 'none') if (scanner !== 'none') { - safeEnqueue(`data: ${JSON.stringify({ status: 'scanning', message: 'Starting vulnerability scan...' })}\n\n`); + sendData({ status: 'scanning', message: 'Starting vulnerability scan...' }); try { const results = await scanImage(image, envId, (progress) => { - safeEnqueue(`data: ${JSON.stringify({ status: 'scan-progress', ...progress })}\n\n`); + sendData({ status: 'scan-progress', ...progress }); }); for (const result of results) { @@ -138,128 +116,99 @@ export const POST: RequestHandler = async (event) => { } const totalVulns = results.reduce((sum, r) => sum + r.vulnerabilities.length, 0); - safeEnqueue(`data: ${JSON.stringify({ + sendData({ status: 'scan-complete', message: `Scan complete - found ${totalVulns} vulnerabilities`, results - })}\n\n`); + }); } catch (scanError) { console.error('Scan-on-pull failed:', scanError); - safeEnqueue(`data: ${JSON.stringify({ + sendData({ status: 'scan-error', error: scanError instanceof Error ? scanError.message : String(scanError) - })}\n\n`); + }); } } }; - const stream = new ReadableStream({ - async start(ctrl) { - controller = ctrl; + // Run operation in background + (async () => { + console.log(`Starting pull for image: ${image}${edgeCheck.isEdge ? ' (edge mode)' : ''}`); - // Start heartbeat to keep connection alive through Traefik (10s idle timeout) - heartbeatInterval = setInterval(() => { - safeEnqueue(`: keepalive\n\n`); - }, 5000); + if (edgeCheck.isEdge && edgeCheck.environmentId) { + if (!isEdgeConnected(edgeCheck.environmentId)) { + sendData({ status: 'error', error: 'Edge agent not connected' }); + failJob(job, 'Edge agent not connected'); + return; + } - console.log(`Starting pull for image: ${image}${edgeCheck.isEdge ? ' (edge mode)' : ''}`); - - // Handle edge mode with streaming - if (edgeCheck.isEdge && edgeCheck.environmentId) { - if (!isEdgeConnected(edgeCheck.environmentId)) { - safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: 'Edge agent not connected' })}\n\n`); - cleanup(); - controller.close(); - return; - } - - const pullUrl = buildPullUrl(image); + const pullUrl = buildPullUrl(image); + const authHeaders = await buildRegistryAuthHeader(image); + await new Promise((resolve) => { const { cancel } = sendEdgeStreamRequest( - edgeCheck.environmentId, + edgeCheck.environmentId!, 'POST', pullUrl, { onData: (data: string) => { - // Data is base64 encoded JSON lines from Docker try { const decoded = Buffer.from(data, 'base64').toString('utf-8'); - // Docker sends newline-delimited JSON - const lines = decoded.split('\n').filter(line => line.trim()); + const lines = decoded.split('\n').filter((line) => line.trim()); for (const line of lines) { try { - const progress = JSON.parse(line); - safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`); + sendData(JSON.parse(line)); } catch { // Ignore parse errors for partial lines } } } catch { - // If not base64, try as-is try { - const progress = JSON.parse(data); - safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`); + sendData(JSON.parse(data)); } catch { // Ignore } } }, onEnd: async () => { - safeEnqueue(`data: ${JSON.stringify({ status: 'complete' })}\n\n`); - - // Handle scan-on-pull + sendData({ status: 'complete' }); await handleScanOnPull(); - - cleanup(); - controller.close(); + completeJob(job, { status: 'complete' }); + resolve(); }, onError: (error: string) => { console.error('Edge pull error:', error); - safeEnqueue(`data: ${JSON.stringify({ status: 'error', error })}\n\n`); - cleanup(); - controller.close(); + sendData({ status: 'error', error }); + failJob(job, error); + resolve(); } - } + }, + undefined, + authHeaders ); - cancelEdgeStream = cancel; - } else { - // Non-edge mode: use existing pullImage function - try { - await pullImage(image, (progress) => { - const data = JSON.stringify(progress) + '\n'; - safeEnqueue(`data: ${data}\n\n`); - }, envId); + // Store cancel reference (not used currently but available) + void cancel; + }); + } else { + try { + await pullImage(image, (progress) => { + sendData(progress); + }, envId); - safeEnqueue(`data: ${JSON.stringify({ status: 'complete' })}\n\n`); - - // Handle scan-on-pull - await handleScanOnPull(); - - cleanup(); - controller.close(); - } catch (error) { - console.error('Error pulling image:', error); - safeEnqueue(`data: ${JSON.stringify({ - status: 'error', - error: String(error) - })}\n\n`); - cleanup(); - controller.close(); - } + sendData({ status: 'complete' }); + await handleScanOnPull(); + completeJob(job, { status: 'complete' }); + } catch (error) { + console.error('Error pulling image:', error); + const errMsg = String(error); + sendData({ status: 'error', error: errMsg }); + failJob(job, errMsg); } - }, - cancel() { - cleanup(); } + })().catch((err) => { + failJob(job, err instanceof Error ? err.message : String(err)); }); - return new Response(stream, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', - 'X-Accel-Buffering': 'no' - } - }); + return json({ jobId: job.id }); }; diff --git a/src/routes/api/images/push/+server.ts b/src/routes/api/images/push/+server.ts index 81bdccd..b029c59 100644 --- a/src/routes/api/images/push/+server.ts +++ b/src/routes/api/images/push/+server.ts @@ -5,6 +5,8 @@ import { getRegistry, getEnvironment } from '$lib/server/db'; import { authorize } from '$lib/server/authorize'; import { auditImage } from '$lib/server/audit'; import { sendEdgeStreamRequest, isEdgeConnected } from '$lib/server/hawser'; +import { prefersJSON } from '$lib/server/sse'; +import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs'; /** * Check if environment is edge mode @@ -119,35 +121,6 @@ export const POST: RequestHandler = async (event) => { // Check if this is an edge environment const edgeCheck = await isEdgeMode(envIdNum); - // Stream the push progress - const encoder = new TextEncoder(); - let controllerClosed = false; - let controller: ReadableStreamDefaultController; - let heartbeatInterval: ReturnType | null = null; - let cancelEdgeStream: (() => void) | null = null; - - const safeEnqueue = (data: string) => { - if (!controllerClosed) { - try { - controller.enqueue(encoder.encode(data)); - } catch { - controllerClosed = true; - } - } - }; - - const cleanup = () => { - if (heartbeatInterval) { - clearInterval(heartbeatInterval); - heartbeatInterval = null; - } - if (cancelEdgeStream) { - cancelEdgeStream(); - cancelEdgeStream = null; - } - controllerClosed = true; - }; - const formatError = (error: any): string => { const errorMessage = error.message || error || ''; let userMessage = errorMessage || 'Failed to push image'; @@ -163,140 +136,87 @@ export const POST: RequestHandler = async (event) => { return userMessage; }; - const stream = new ReadableStream({ - async start(ctrl) { - controller = ctrl; + // Core push logic — emit callback receives progress data objects + async function runPush(emit: (data: unknown) => void): Promise { + emit({ status: 'tagging', message: 'Tagging image...' }); + await tagImage(imageId, repo, tag, envIdNum); + emit({ status: 'pushing', message: 'Pushing to registry...' }); - // Start heartbeat to keep connection alive through Traefik (10s idle timeout) - heartbeatInterval = setInterval(() => { - safeEnqueue(`: keepalive\n\n`); - }, 5000); + if (edgeCheck.isEdge && edgeCheck.environmentId) { + if (!isEdgeConnected(edgeCheck.environmentId)) { + emit({ status: 'error', error: 'Edge agent not connected' }); + return; + } - try { - // Send tagging status - safeEnqueue(`data: ${JSON.stringify({ status: 'tagging', message: 'Tagging image...' })}\n\n`); + const authHeader = Buffer.from(JSON.stringify(authConfig)).toString('base64'); - // Tag the image with the target registry - await tagImage(imageId, repo, tag, envIdNum); - - // Send pushing status - safeEnqueue(`data: ${JSON.stringify({ status: 'pushing', message: 'Pushing to registry...' })}\n\n`); - - // Handle edge mode with streaming - if (edgeCheck.isEdge && edgeCheck.environmentId) { - if (!isEdgeConnected(edgeCheck.environmentId)) { - safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: 'Edge agent not connected' })}\n\n`); - cleanup(); - controller.close(); - return; - } - - // Create X-Registry-Auth header - const authHeader = Buffer.from(JSON.stringify(authConfig)).toString('base64'); - - const { cancel } = sendEdgeStreamRequest( - edgeCheck.environmentId, - 'POST', - `/images/${encodeURIComponent(targetTag)}/push`, - { - onData: (data: string) => { - // Data is base64 encoded JSON lines from Docker - try { - const decoded = Buffer.from(data, 'base64').toString('utf-8'); - const lines = decoded.split('\n').filter(line => line.trim()); - for (const line of lines) { - try { - const progress = JSON.parse(line); - if (progress.error) { - safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: formatError(progress.error) })}\n\n`); - } else { - safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`); - } - } catch { - // Ignore parse errors for partial lines - } - } - } catch { - // If not base64, try as-is + await new Promise((resolve, reject) => { + sendEdgeStreamRequest( + edgeCheck.environmentId!, + 'POST', + `/images/${encodeURIComponent(targetTag)}/push`, + { + onData: (data: string) => { + try { + const decoded = Buffer.from(data, 'base64').toString('utf-8'); + for (const line of decoded.split('\n').filter((l) => l.trim())) { try { - const progress = JSON.parse(data); - if (progress.error) { - safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: formatError(progress.error) })}\n\n`); - } else { - safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`); - } - } catch { - // Ignore - } + const progress = JSON.parse(line); + emit(progress.error ? { status: 'error', error: formatError(progress.error) } : progress); + } catch { /* ignore partial lines */ } } - }, - onEnd: async () => { - // Audit log - await auditImage(event, 'push', imageId, imageName || targetTag, envIdNum, { targetTag, registry: registry.name }); - - safeEnqueue(`data: ${JSON.stringify({ - status: 'complete', - message: `Image pushed to ${targetTag}`, - targetTag - })}\n\n`); - - cleanup(); - controller.close(); - }, - onError: (error: string) => { - console.error('Edge push error:', error); - safeEnqueue(`data: ${JSON.stringify({ status: 'error', error: formatError(error) })}\n\n`); - cleanup(); - controller.close(); + } catch { + try { + const progress = JSON.parse(data); + emit(progress.error ? { status: 'error', error: formatError(progress.error) } : progress); + } catch { /* ignore */ } } }, - undefined, - { 'X-Registry-Auth': authHeader } - ); - - cancelEdgeStream = cancel; - } else { - // Non-edge mode: use existing pushImage function - await pushImage(targetTag, authConfig, (progress) => { - safeEnqueue(`data: ${JSON.stringify(progress)}\n\n`); - }, envIdNum); - - // Audit log - await auditImage(event, 'push', imageId, imageName || targetTag, envIdNum, { targetTag, registry: registry.name }); - - // Send completion message - safeEnqueue(`data: ${JSON.stringify({ - status: 'complete', - message: `Image pushed to ${targetTag}`, - targetTag - })}\n\n`); - - cleanup(); - controller.close(); - } - } catch (error: any) { - console.error('Error pushing image:', error); - safeEnqueue(`data: ${JSON.stringify({ - status: 'error', - error: formatError(error) - })}\n\n`); - cleanup(); - controller.close(); - } - }, - cancel() { - cleanup(); + onEnd: async () => { + await auditImage(event, 'push', imageId, imageName || targetTag, envIdNum, { targetTag, registry: registry.name }); + emit({ status: 'complete', message: `Image pushed to ${targetTag}`, targetTag }); + resolve(); + }, + onError: (error: string) => { + console.error('Edge push error:', error); + emit({ status: 'error', error: formatError(error) }); + reject(new Error(error)); + } + }, + undefined, + { 'X-Registry-Auth': authHeader } + ); + }); + } else { + await pushImage(targetTag, authConfig, (progress) => emit(progress), envIdNum); + await auditImage(event, 'push', imageId, imageName || targetTag, envIdNum, { targetTag, registry: registry.name }); + emit({ status: 'complete', message: `Image pushed to ${targetTag}`, targetTag }); } - }); + } - return new Response(stream, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', - 'X-Accel-Buffering': 'no' + // Sync path for API clients sending Accept: application/json only + if (prefersJSON(request)) { + try { + let lastEvent: unknown = null; + await runPush((data) => { lastEvent = data; }); + return json(lastEvent || { success: true }); + } catch (error: any) { + return json({ status: 'error', error: formatError(error) }, { status: 500 }); } - }); + } + + // Job pattern: return jobId immediately, push runs in background + const job = createJob(); + (async () => { + try { + await runPush((data) => appendLine(job, { data })); + completeJob(job, job.lines[job.lines.length - 1]?.data ?? { success: true }); + } catch (error: any) { + appendLine(job, { data: { status: 'error', error: formatError(error) } }); + failJob(job, error.message); + } + })(); + return json({ jobId: job.id }); } catch (error: any) { console.error('Error setting up push:', error); return json({ error: error.message || 'Failed to push image' }, { status: 500 }); diff --git a/src/routes/api/images/scan/+server.ts b/src/routes/api/images/scan/+server.ts index ae503dc..7d6ba12 100644 --- a/src/routes/api/images/scan/+server.ts +++ b/src/routes/api/images/scan/+server.ts @@ -2,6 +2,7 @@ import { json, type RequestHandler } from '@sveltejs/kit'; import { scanImage, type ScanProgress, type ScanResult } from '$lib/server/scanner'; import { saveVulnerabilityScan, getLatestScanForImage } from '$lib/server/db'; import { authorize } from '$lib/server/authorize'; +import { createJob, appendLine, completeJob, failJob } from '$lib/server/jobs'; // Helper to convert ScanResult to database format function scanResultToDbFormat(result: ScanResult, envId?: number) { @@ -23,7 +24,7 @@ function scanResultToDbFormat(result: ScanResult, envId?: number) { }; } -// POST - Start a scan (returns SSE stream for progress) +// POST - Start a scan (returns { jobId } for progress polling) export const POST: RequestHandler = async ({ request, url, cookies }) => { const auth = await authorize(cookies); @@ -42,75 +43,47 @@ export const POST: RequestHandler = async ({ request, url, cookies }) => { return json({ error: 'Image name is required' }, { status: 400 }); } - // Create a readable stream for SSE - const stream = new ReadableStream({ - async start(controller) { - const encoder = new TextEncoder(); - let controllerClosed = false; + // Job pattern: create job, run in background, return jobId immediately + const job = createJob(); - const sendProgress = (progress: ScanProgress) => { - if (controllerClosed) return; - try { - const data = `data: ${JSON.stringify(progress)}\n\n`; - controller.enqueue(encoder.encode(data)); - } catch { - controllerClosed = true; - } - }; + const sendProgress = (progress: ScanProgress) => { + appendLine(job, { data: progress }); + }; - // Send SSE keepalive comments every 5s to prevent Traefik timeout - const keepaliveInterval = setInterval(() => { - if (controllerClosed) return; - try { - controller.enqueue(encoder.encode(`: keepalive\n\n`)); - } catch { - controllerClosed = true; - } - }, 5000); + (async () => { + try { + const results = await scanImage(imageName, envId, sendProgress, forceScannerType); - try { - const results = await scanImage(imageName, envId, sendProgress, forceScannerType); - - // Save results to database - for (const result of results) { - await saveVulnerabilityScan(scanResultToDbFormat(result, envId)); - } - - // Send final complete message with all results - sendProgress({ - stage: 'complete', - message: `Scan complete - found ${results.reduce((sum, r) => sum + r.vulnerabilities.length, 0)} vulnerabilities`, - progress: 100, - result: results[0], - results: results // Include all scanner results - }); - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - sendProgress({ - stage: 'error', - message: `Scan failed: ${errorMsg}`, - error: errorMsg - }); - } finally { - clearInterval(keepaliveInterval); - if (!controllerClosed) { - try { - controller.close(); - } catch { - // Already closed - } - } + // Save results to database + for (const result of results) { + await saveVulnerabilityScan(scanResultToDbFormat(result, envId)); } + + // Send final complete message with all results + const completeProgress: ScanProgress = { + stage: 'complete', + message: `Scan complete - found ${results.reduce((sum, r) => sum + r.vulnerabilities.length, 0)} vulnerabilities`, + progress: 100, + result: results[0], + results: results // Include all scanner results + }; + sendProgress(completeProgress); + completeJob(job, completeProgress); + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + const errorProgress: ScanProgress = { + stage: 'error', + message: `Scan failed: ${errorMsg}`, + error: errorMsg + }; + sendProgress(errorProgress); + failJob(job, errorMsg); } + })().catch((err) => { + failJob(job, err instanceof Error ? err.message : String(err)); }); - return new Response(stream, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive' - } - }); + return json({ jobId: job.id }); }; // GET - Get cached scan results for an image diff --git a/src/routes/api/jobs/[id]/+server.ts b/src/routes/api/jobs/[id]/+server.ts new file mode 100644 index 0000000..f1a624b --- /dev/null +++ b/src/routes/api/jobs/[id]/+server.ts @@ -0,0 +1,23 @@ +import { json } from '@sveltejs/kit'; +import { getJob } from '$lib/server/jobs'; +import type { RequestHandler } from './$types'; + +/** + * GET /api/jobs/[id] + * Poll a job's status and accumulated lines. + * Returns all lines every time — client tracks its own cursor locally. + * No auth required: job IDs are UUIDs (unguessable), no sensitive data beyond what the initiating user triggered. + */ +export const GET: RequestHandler = async ({ params }) => { + const job = getJob(params.id); + if (!job) { + return json({ error: 'Job not found' }, { status: 404 }); + } + + return json({ + id: job.id, + status: job.status, + lines: job.lines, + result: job.result ?? null + }); +}; diff --git a/src/routes/api/logs/merged/+server.ts b/src/routes/api/logs/merged/+server.ts index 51492aa..8480c6b 100644 --- a/src/routes/api/logs/merged/+server.ts +++ b/src/routes/api/logs/merged/+server.ts @@ -1,6 +1,8 @@ import type { RequestHandler } from './$types'; import { authorize } from '$lib/server/authorize'; import { getEnvironment } from '$lib/server/db'; +import { unixSocketRequest, unixSocketStreamRequest, httpsAgentRequest } from '$lib/server/docker'; +import type { DockerClientConfig as BaseDockerClientConfig } from '$lib/server/docker'; import { sendEdgeRequest, sendEdgeStreamRequest, isEdgeConnected } from '$lib/server/hawser'; import { existsSync } from 'node:fs'; import { homedir } from 'node:os'; @@ -424,37 +426,20 @@ export const GET: RequestHandler = async ({ url, cookies }) => { let inspectResponse: Response; if (config.type === 'socket') { - inspectResponse = await fetch(`http://localhost${inspectPath}`, { - // @ts-ignore - Bun supports unix socket - unix: config.socketPath - }); + inspectResponse = await unixSocketRequest(config.socketPath, inspectPath); + } else if (config.type === 'https') { + const extraHeaders: Record = {}; + if (config.hawserToken) extraHeaders['X-Hawser-Token'] = config.hawserToken; + inspectResponse = await httpsAgentRequest(config as BaseDockerClientConfig, inspectPath, {}, false, extraHeaders); } else { - const inspectUrl = `${config.type}://${config.host}:${config.port}${inspectPath}`; + const inspectUrl = `http://${config.host}:${config.port}${inspectPath}`; const inspectHeaders: Record = {}; if (config.hawserToken) inspectHeaders['X-Hawser-Token'] = config.hawserToken; - - // Build fetch options - only include tls for HTTPS - const fetchOptions: any = { - headers: inspectHeaders, - signal: AbortSignal.timeout(30000) - }; - if (config.type === 'https') { - fetchOptions.tls = { - sessionTimeout: 0, - servername: config.host, - rejectUnauthorized: !config.skipVerify - }; - if (config.ca) fetchOptions.tls.ca = [config.ca]; - if (config.cert) fetchOptions.tls.cert = [config.cert]; - if (config.key) fetchOptions.tls.key = config.key; - fetchOptions.keepalive = false; - if (process.env.DEBUG_TLS) fetchOptions.verbose = true; - } - - inspectResponse = await fetch(inspectUrl, fetchOptions); + inspectResponse = await fetch(inspectUrl, { headers: inspectHeaders, signal: AbortSignal.timeout(30000) }); } if (!inspectResponse.ok) { + await inspectResponse.arrayBuffer().catch(() => {}); console.log(`[merged-logs] Inspect failed for ${containerId.slice(0, 12)}, skipping`); return null; } @@ -468,39 +453,20 @@ export const GET: RequestHandler = async ({ url, cookies }) => { let logsResponse: Response; if (config.type === 'socket') { - logsResponse = await fetch(`http://localhost${logsPath}`, { - // @ts-ignore - Bun supports unix socket - unix: config.socketPath, - signal: abortController.signal - }); + logsResponse = await unixSocketStreamRequest(config.socketPath, logsPath); + } else if (config.type === 'https') { + const extraHeaders: Record = {}; + if (config.hawserToken) extraHeaders['X-Hawser-Token'] = config.hawserToken; + logsResponse = await httpsAgentRequest(config as BaseDockerClientConfig, logsPath, {}, true, extraHeaders); } else { - const logsUrl = `${config.type}://${config.host}:${config.port}${logsPath}`; + const logsUrl = `http://${config.host}:${config.port}${logsPath}`; const logsHeaders: Record = {}; if (config.hawserToken) logsHeaders['X-Hawser-Token'] = config.hawserToken; - - // For logs streaming, use the cleanup abort controller without a timeout - // (the stream needs to stay open indefinitely) - const fetchOptions: any = { - headers: logsHeaders, - signal: abortController.signal - }; - if (config.type === 'https') { - fetchOptions.tls = { - sessionTimeout: 0, - servername: config.host, - rejectUnauthorized: !config.skipVerify - }; - if (config.ca) fetchOptions.tls.ca = [config.ca]; - if (config.cert) fetchOptions.tls.cert = [config.cert]; - if (config.key) fetchOptions.tls.key = config.key; - fetchOptions.keepalive = false; - if (process.env.DEBUG_TLS) fetchOptions.verbose = true; - } - - logsResponse = await fetch(logsUrl, fetchOptions); + logsResponse = await fetch(logsUrl, { headers: logsHeaders, signal: abortController.signal }); } if (!logsResponse.ok) { + await logsResponse.arrayBuffer().catch(() => {}); console.error(`[merged-logs] Failed to get logs for container ${containerId}: ${logsResponse.status}`); return null; } @@ -647,7 +613,8 @@ export const GET: RequestHandler = async ({ url, cookies }) => { for (const source of sources) { if (source.reader) { try { - source.reader.releaseLock(); + await source.reader.cancel().catch(() => {}); + source.reader.releaseLock(); } catch { // Ignore } diff --git a/src/routes/api/metrics/+server.ts b/src/routes/api/metrics/+server.ts deleted file mode 100644 index 266fbd8..0000000 --- a/src/routes/api/metrics/+server.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { json } from '@sveltejs/kit'; -import type { RequestHandler } from './$types'; -import { getHostMetrics } from '$lib/server/db'; - -export const GET: RequestHandler = async ({ url }) => { - try { - const limit = parseInt(url.searchParams.get('limit') || '60'); - const envId = url.searchParams.get('env'); - const envIdNum = envId ? parseInt(envId) : undefined; - - const metrics = await getHostMetrics(limit, envIdNum); - - // Return metrics in chronological order (oldest first) for graphing - const chronological = metrics.reverse(); - - return json({ - metrics: chronological, - latest: metrics.length > 0 ? metrics[metrics.length - 1] : null - }); - } catch (error) { - console.error('Failed to get host metrics:', error); - return json({ error: 'Failed to get host metrics' }, { status: 500 }); - } -}; diff --git a/src/routes/api/networks/+server.ts b/src/routes/api/networks/+server.ts index b00ffc0..02c1f1f 100644 --- a/src/routes/api/networks/+server.ts +++ b/src/routes/api/networks/+server.ts @@ -1,6 +1,6 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -import { listNetworks, createNetwork, EnvironmentNotFoundError, type CreateNetworkOptions } from '$lib/server/docker'; +import { listNetworks, createNetwork, EnvironmentNotFoundError, DockerConnectionError, type CreateNetworkOptions } from '$lib/server/docker'; import { authorize } from '$lib/server/authorize'; import { auditNetwork } from '$lib/server/audit'; import { hasEnvironments } from '$lib/server/db'; @@ -33,7 +33,9 @@ export const GET: RequestHandler = async ({ url, cookies }) => { if (error instanceof EnvironmentNotFoundError) { return json({ error: 'Environment not found' }, { status: 404 }); } - console.error('Failed to list networks:', error); + if (!(error instanceof DockerConnectionError)) { + console.error('Failed to list networks:', error); + } return json({ error: 'Failed to list networks' }, { status: 500 }); } }; diff --git a/src/routes/api/prune/images/+server.ts b/src/routes/api/prune/images/+server.ts index eb6c7ee..92a5abe 100644 --- a/src/routes/api/prune/images/+server.ts +++ b/src/routes/api/prune/images/+server.ts @@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit'; import { pruneImages } from '$lib/server/docker'; import { authorize } from '$lib/server/authorize'; import { audit } from '$lib/server/audit'; +import { createJobResponse } from '$lib/server/sse'; import type { RequestHandler } from './$types'; export const POST: RequestHandler = async (event) => { @@ -17,19 +18,21 @@ export const POST: RequestHandler = async (event) => { return json({ error: 'Permission denied' }, { status: 403 }); } - try { - const result = await pruneImages(danglingOnly, envIdNum); + return createJobResponse(async (send) => { + try { + const result = await pruneImages(danglingOnly, envIdNum); - // Audit log - await audit(event, 'prune', 'image', { - environmentId: envIdNum, - description: `Pruned ${danglingOnly ? 'dangling' : 'unused'} images`, - details: { danglingOnly, result } - }); + // Audit log + await audit(event, 'prune', 'image', { + environmentId: envIdNum, + description: `Pruned ${danglingOnly ? 'dangling' : 'unused'} images`, + details: { danglingOnly, result } + }); - return json({ success: true, result }); - } catch (error) { - console.error('Error pruning images:', error); - return json({ error: 'Failed to prune images' }, { status: 500 }); - } + send('result', { success: true, result }); + } catch (error) { + console.error('Error pruning images:', error); + send('result', { success: false, error: 'Failed to prune images' }); + } + }, event.request); }; diff --git a/src/routes/api/schedules/stream/+server.ts b/src/routes/api/schedules/stream/+server.ts index 8bcfc79..597c36b 100644 --- a/src/routes/api/schedules/stream/+server.ts +++ b/src/routes/api/schedules/stream/+server.ts @@ -340,7 +340,8 @@ export const GET: RequestHandler = async ({ cookies }) => { headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive' + 'Connection': 'keep-alive', + 'X-Accel-Buffering': 'no' } }); }; diff --git a/src/routes/api/self-update/+server.ts b/src/routes/api/self-update/+server.ts index 2d8cc40..5c44f7c 100644 --- a/src/routes/api/self-update/+server.ts +++ b/src/routes/api/self-update/+server.ts @@ -1,22 +1,22 @@ import { json } from '@sveltejs/kit'; import { authorize } from '$lib/server/authorize'; import { getOwnContainerId, getHostDockerSocket } from '$lib/server/host-path'; +import { buildRegistryAuthHeader, unixSocketRequest, unixSocketStreamRequest } from '$lib/server/docker'; import type { RequestHandler } from './$types'; +import { prefersJSON, sseToJSON } from '$lib/server/sse'; const UPDATER_IMAGE = 'fnsys/dockhand-updater:latest'; const UPDATER_LABEL = 'dockhand.updater'; +const DOCKER_SOCKET = process.env.DOCKER_SOCKET || '/var/run/docker.sock'; -/** - * Fetch from the local Docker socket directly. - * Self-update always operates on the local engine — no environment routing needed. - */ -async function localDockerFetch(path: string, options: RequestInit = {}): Promise { - const socketPath = process.env.DOCKER_SOCKET || '/var/run/docker.sock'; - return fetch(`http://localhost${path}`, { - ...options, - // @ts-ignore - Bun supports unix sockets - unix: socketPath - }); +/** Fetch from the local Docker socket (buffered). */ +function localDockerFetch(path: string, options: RequestInit = {}): Promise { + return unixSocketRequest(DOCKER_SOCKET, path, options); +} + +/** Fetch from the local Docker socket (streaming body for pull progress). */ +function localDockerStreamFetch(path: string, options: RequestInit = {}): Promise { + return unixSocketStreamRequest(DOCKER_SOCKET, path, options); } /** @@ -34,9 +34,10 @@ async function pullImageLocal(imageName: string, onProgress?: (line: string) => } } - const response = await localDockerFetch( + const authHeaders = await buildRegistryAuthHeader(imageName); + const response = await localDockerStreamFetch( `/images/create?fromImage=${encodeURIComponent(fromImage)}&tag=${encodeURIComponent(tag)}`, - { method: 'POST' } + { method: 'POST', headers: authHeaders } ); if (!response.ok) { @@ -400,11 +401,14 @@ export const POST: RequestHandler = async ({ request, cookies }) => { } }); - return new Response(stream, { + const sseResponse = new Response(stream, { headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive' + 'Connection': 'keep-alive', + 'X-Accel-Buffering': 'no' } }); + if (prefersJSON(request)) return sseToJSON(sseResponse); + return sseResponse; }; diff --git a/src/routes/api/self-update/check/+server.ts b/src/routes/api/self-update/check/+server.ts index 50d9700..c81cee8 100644 --- a/src/routes/api/self-update/check/+server.ts +++ b/src/routes/api/self-update/check/+server.ts @@ -1,19 +1,15 @@ import { json } from '@sveltejs/kit'; import { authorize } from '$lib/server/authorize'; import { getOwnContainerId } from '$lib/server/host-path'; -import { getRegistryManifestDigest } from '$lib/server/docker'; +import { getRegistryManifestDigest, unixSocketRequest } from '$lib/server/docker'; +import { compareVersions } from '$lib/utils/version'; import type { RequestHandler } from './$types'; -/** - * Fetch from the local Docker socket directly (not through environment routing) - */ -async function localDockerFetch(path: string, options: RequestInit = {}): Promise { - const socketPath = process.env.DOCKER_SOCKET || '/var/run/docker.sock'; - return fetch(`http://localhost${path}`, { - ...options, - // @ts-ignore - Bun supports unix sockets - unix: socketPath - }); +const DOCKER_SOCKET = process.env.DOCKER_SOCKET || '/var/run/docker.sock'; + +/** Fetch from the local Docker socket directly (not through environment routing) */ +function localDockerFetch(path: string, options: RequestInit = {}): Promise { + return unixSocketRequest(DOCKER_SOCKET, path, options); } /** @@ -78,6 +74,92 @@ export const GET: RequestHandler = async ({ cookies }) => { }); } + // Extract tag from image name + const colonIdx = currentImage.lastIndexOf(':'); + const tag = colonIdx > -1 ? currentImage.substring(colonIdx + 1) : 'latest'; + const imageWithoutTag = colonIdx > -1 ? currentImage.substring(0, colonIdx) : currentImage; + + // Check if this is a versioned tag (e.g., v1.0.18, 1.0.18, v1.0.18-baseline) + const versionMatch = tag.match(/^(v?\d+\.\d+\.\d+)(-baseline)?$/); + + if (versionMatch) { + // Version-based check: compare against latest released version from changelog + const currentTagVersion = versionMatch[1]; + const suffix = versionMatch[2] || ''; // '-baseline' or '' + + try { + const changelogResponse = await fetch( + 'https://raw.githubusercontent.com/Finsys/dockhand/main/src/lib/data/changelog.json', + { signal: AbortSignal.timeout(5000) } + ); + + if (!changelogResponse.ok) { + return json({ + updateAvailable: false, + currentImage, + containerName, + isComposeManaged, + error: 'Could not fetch changelog from GitHub' + }); + } + + const changelog = await changelogResponse.json() as Array<{ + version: string; + comingSoon?: boolean; + date?: string; + changes?: Array<{ type: string; text: string }>; + }>; + + // Find latest released version (first entry without comingSoon) + const latestRelease = changelog.find(entry => !entry.comingSoon); + + if (!latestRelease) { + return json({ + updateAvailable: false, + currentImage, + containerName, + isComposeManaged, + error: 'No released version found in changelog' + }); + } + + const latestVersion = latestRelease.version; + const hasNewer = compareVersions(latestVersion, currentTagVersion) > 0; + + if (hasNewer) { + // Build new image tag preserving registry prefix and suffix + const newTag = `v${latestVersion.replace(/^v/, '')}${suffix}`; + const newImage = `${imageWithoutTag}:${newTag}`; + + return json({ + updateAvailable: true, + currentImage, + newImage, + latestVersion: latestVersion.replace(/^v/, ''), + containerName, + isComposeManaged + }); + } + + return json({ + updateAvailable: false, + currentImage, + containerName, + isComposeManaged + }); + } catch (err) { + return json({ + updateAvailable: false, + currentImage, + containerName, + isComposeManaged, + error: 'Version check failed: ' + String(err) + }); + } + } + + // Digest-based check for mutable tags (:latest, :baseline, etc.) + // Inspect image via local Docker socket to get RepoDigests const imageResponse = await localDockerFetch(`/images/${encodeURIComponent(currentImageId)}/json`); if (!imageResponse.ok) { @@ -105,6 +187,7 @@ export const GET: RequestHandler = async ({ cookies }) => { return json({ updateAvailable: false, currentImage, + newImage: currentImage, containerName, isComposeManaged, isLocalImage: true @@ -117,6 +200,7 @@ export const GET: RequestHandler = async ({ cookies }) => { return json({ updateAvailable: false, currentImage, + newImage: currentImage, containerName, isComposeManaged, error: 'Could not query registry' @@ -128,6 +212,7 @@ export const GET: RequestHandler = async ({ cookies }) => { return json({ updateAvailable: hasUpdate, currentImage, + newImage: currentImage, currentDigest: localDigests[0], newDigest: registryDigest, containerName, diff --git a/src/routes/api/self-update/progress/+server.ts b/src/routes/api/self-update/progress/+server.ts index 7f2d60d..d92bd9b 100644 --- a/src/routes/api/self-update/progress/+server.ts +++ b/src/routes/api/self-update/progress/+server.ts @@ -1,16 +1,13 @@ import { json } from '@sveltejs/kit'; import { authorize } from '$lib/server/authorize'; +import { unixSocketRequest } from '$lib/server/docker'; import type { RequestHandler } from './$types'; -/** - * Fetch from the local Docker socket directly - */ -async function localDockerFetch(path: string): Promise { - const socketPath = process.env.DOCKER_SOCKET || '/var/run/docker.sock'; - return fetch(`http://localhost${path}`, { - // @ts-ignore - Bun supports unix sockets - unix: socketPath - }); +const DOCKER_SOCKET = process.env.DOCKER_SOCKET || '/var/run/docker.sock'; + +/** Fetch from the local Docker socket directly */ +function localDockerFetch(path: string): Promise { + return unixSocketRequest(DOCKER_SOCKET, path); } /** diff --git a/src/routes/api/settings/general/+server.ts b/src/routes/api/settings/general/+server.ts index b49362c..aaf3ecc 100644 --- a/src/routes/api/settings/general/+server.ts +++ b/src/routes/api/settings/general/+server.ts @@ -29,7 +29,7 @@ import { } from '$lib/server/db'; import { authorize } from '$lib/server/authorize'; import { refreshSystemJobs } from '$lib/server/scheduler'; -import { sendToEventSubprocess, sendToMetricsSubprocess, type UpdateIntervalCommand } from '$lib/server/subprocess-manager'; +import { sendToEventSubprocess, sendToMetricsSubprocess } from '$lib/server/subprocess-manager'; export type TimeFormat = '12h' | '24h'; export type DateFormat = 'MM/DD/YYYY' | 'DD/MM/YYYY' | 'YYYY-MM-DD' | 'DD.MM.YYYY'; diff --git a/src/routes/api/stacks/+server.ts b/src/routes/api/stacks/+server.ts index e155c9b..8573bc8 100644 --- a/src/routes/api/stacks/+server.ts +++ b/src/routes/api/stacks/+server.ts @@ -1,9 +1,10 @@ import { json } from '@sveltejs/kit'; import { listComposeStacks, deployStack, saveStackComposeFile, writeStackEnvFile, writeRawStackEnvFile, saveStackEnvVarsToDb } from '$lib/server/stacks'; -import { EnvironmentNotFoundError } from '$lib/server/docker'; +import { EnvironmentNotFoundError, DockerConnectionError } from '$lib/server/docker'; import { upsertStackSource, getStackSources } from '$lib/server/db'; import { authorize } from '$lib/server/authorize'; import { auditStack } from '$lib/server/audit'; +import { createJobResponse } from '$lib/server/sse'; import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ url, cookies }) => { @@ -62,8 +63,11 @@ export const GET: RequestHandler = async ({ url, cookies }) => { if (error instanceof EnvironmentNotFoundError) { return json({ error: 'Environment not found' }, { status: 404 }); } + // Silently return empty for connection errors (offline environments) + if (error instanceof DockerConnectionError) { + return json([]); + } console.error('Error listing compose stacks:', error); - // Return empty array instead of error to allow UI to load return json([]); } }; @@ -162,20 +166,7 @@ export const POST: RequestHandler = async (event) => { } } - // Deploy and start the stack - const result = await deployStack({ - name, - compose, - envId: envIdNum, - composePath: composePath || undefined, - envPath: envPath || undefined - }); - - if (!result.success) { - return json({ error: result.error, output: result.output }, { status: 400 }); - } - - // Record the stack as internally created with custom paths if provided + // Record the stack in DB before deploying - ensures it exists even if deploy fails await upsertStackSource({ stackName: name, environmentId: envIdNum, @@ -184,10 +175,31 @@ export const POST: RequestHandler = async (event) => { envPath: envPath || undefined }); - // Audit log (create + deploy in one action) - await auditStack(event, 'deploy', name, envIdNum); + // Deploy via SSE to keep connection alive during long operations + return createJobResponse(async (send) => { + try { + const result = await deployStack({ + name, + compose, + envId: envIdNum, + composePath: composePath || undefined, + envPath: envPath || undefined + }); - return json({ success: true, started: true, output: result.output }); + if (!result.success) { + send('result', { success: false, error: result.error, output: result.output }); + return; + } + + // Audit log (create + deploy in one action) + await auditStack(event, 'deploy', name, envIdNum); + + send('result', { success: true, started: true, output: result.output }); + } catch (error: any) { + console.error('Error deploying compose stack:', error); + send('result', { success: false, error: error.message || 'Failed to deploy stack' }); + } + }, request); } catch (error: any) { console.error('Error creating compose stack:', error); return json({ error: error.message || 'Failed to create stack' }, { status: 500 }); diff --git a/src/routes/api/stacks/[name]/compose/+server.ts b/src/routes/api/stacks/[name]/compose/+server.ts index 2bb3093..d68666b 100644 --- a/src/routes/api/stacks/[name]/compose/+server.ts +++ b/src/routes/api/stacks/[name]/compose/+server.ts @@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import { getStackComposeFile, deployStack, saveStackComposeFile } from '$lib/server/stacks'; import { authorize } from '$lib/server/authorize'; +import { createJobResponse } from '$lib/server/sse'; // GET /api/stacks/[name]/compose - Get compose file content export const GET: RequestHandler = async ({ params, url, cookies }) => { @@ -66,7 +67,6 @@ export const PUT: RequestHandler = async ({ params, request, url, cookies }) => ? { composePath, envPath, moveFromDir, oldComposePath, oldEnvPath } : undefined; - let result; if (restart) { // Deploy with docker compose up -d --force-recreate // Force recreate ensures env var changes are applied @@ -79,19 +79,34 @@ export const PUT: RequestHandler = async ({ params, request, url, cookies }) => } // Get authoritative paths from DB/filesystem for deploy const composeInfo = await getStackComposeFile(name, envIdNum); - result = await deployStack({ - name, - compose: content, - envId: envIdNum, - forceRecreate: true, - composePath: composeInfo.composePath || undefined, - envPath: composeInfo.envPath || undefined - }); - } else { - // Just save the file without restarting (update operation, not create) - result = await saveStackComposeFile(name, content, false, envIdNum, pathOptions); + + // Deploy via SSE to keep connection alive during long operations + return createJobResponse(async (send) => { + try { + const result = await deployStack({ + name, + compose: content, + envId: envIdNum, + forceRecreate: true, + composePath: composeInfo.composePath || undefined, + envPath: composeInfo.envPath || undefined + }); + + if (!result.success) { + send('result', { success: false, error: result.error }); + return; + } + send('result', { success: true }); + } catch (error: any) { + console.error(`Error deploying stack ${name}:`, error); + send('result', { success: false, error: error.message || 'Failed to deploy stack' }); + } + }, request); } + // Just save the file without restarting (update operation, not create) + const result = await saveStackComposeFile(name, content, false, envIdNum, pathOptions); + if (!result.success) { return json({ error: result.error }, { status: 500 }); } diff --git a/src/routes/api/stacks/[name]/down/+server.ts b/src/routes/api/stacks/[name]/down/+server.ts index 19f6aa0..e3cd648 100644 --- a/src/routes/api/stacks/[name]/down/+server.ts +++ b/src/routes/api/stacks/[name]/down/+server.ts @@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit'; import { downStack, ComposeFileNotFoundError } from '$lib/server/stacks'; import { authorize } from '$lib/server/authorize'; import { auditStack } from '$lib/server/audit'; +import { createJobResponse } from '$lib/server/sse'; import type { RequestHandler } from './$types'; export const POST: RequestHandler = async (event) => { @@ -21,31 +22,35 @@ export const POST: RequestHandler = async (event) => { return json({ error: 'Access denied to this environment' }, { status: 403 }); } + // Parse body BEFORE creating SSE response (body can only be read once) + let removeVolumes = false; try { - // Parse body for optional removeVolumes flag - let removeVolumes = false; - try { - const body = await request.json(); - removeVolumes = body.removeVolumes === true; - } catch { - // No body or invalid JSON - use defaults - } - - const stackName = decodeURIComponent(params.name); - const result = await downStack(stackName, envIdNum, removeVolumes); - - // Audit log - await auditStack(event, 'down', stackName, envIdNum, { removeVolumes }); - - if (!result.success) { - return json({ success: false, error: result.error }, { status: 400 }); - } - return json({ success: true, output: result.output }); - } catch (error) { - if (error instanceof ComposeFileNotFoundError) { - return json({ error: error.message }, { status: 404 }); - } - console.error('Error downing compose stack:', error); - return json({ error: 'Failed to down compose stack' }, { status: 500 }); + const body = await request.json(); + removeVolumes = body.removeVolumes === true; + } catch { + // No body or invalid JSON - use defaults } + + return createJobResponse(async (send) => { + try { + const stackName = decodeURIComponent(params.name); + const result = await downStack(stackName, envIdNum, removeVolumes); + + // Audit log + await auditStack(event, 'down', stackName, envIdNum, { removeVolumes }); + + if (!result.success) { + send('result', { success: false, error: result.error }); + return; + } + send('result', { success: true, output: result.output }); + } catch (error) { + if (error instanceof ComposeFileNotFoundError) { + send('result', { success: false, error: error.message }); + return; + } + console.error('Error downing compose stack:', error); + send('result', { success: false, error: 'Failed to down compose stack' }); + } + }, request); }; diff --git a/src/routes/api/stacks/[name]/env/+server.ts b/src/routes/api/stacks/[name]/env/+server.ts index a4c1ae5..df03759 100644 --- a/src/routes/api/stacks/[name]/env/+server.ts +++ b/src/routes/api/stacks/[name]/env/+server.ts @@ -2,7 +2,7 @@ import { json } from '@sveltejs/kit'; import { getStackEnvVars, setStackEnvVars, getStackSource } from '$lib/server/db'; import { findStackDir } from '$lib/server/stacks'; import { authorize } from '$lib/server/authorize'; -import { existsSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { join, dirname } from 'node:path'; import type { RequestHandler } from './$types'; @@ -94,7 +94,7 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { // Internal/adopted stacks: non-secrets from file, secrets from DB if (envFilePath && existsSync(envFilePath)) { try { - const content = await Bun.file(envFilePath).text(); + const content = readFileSync(envFilePath, 'utf-8'); const fileVars = parseEnvFile(content); for (const [key, value] of Object.entries(fileVars)) { variables.push({ key, value, isSecret: false }); diff --git a/src/routes/api/stacks/[name]/env/raw/+server.ts b/src/routes/api/stacks/[name]/env/raw/+server.ts index edd39fe..60b09d0 100644 --- a/src/routes/api/stacks/[name]/env/raw/+server.ts +++ b/src/routes/api/stacks/[name]/env/raw/+server.ts @@ -2,7 +2,7 @@ import { json } from '@sveltejs/kit'; import { findStackDir, getStackDir } from '$lib/server/stacks'; import { getStackSource } from '$lib/server/db'; import { authorize } from '$lib/server/authorize'; -import { existsSync, rmSync } from 'node:fs'; +import { existsSync, rmSync, readFileSync, writeFileSync } from 'node:fs'; import { join, dirname } from 'node:path'; import type { RequestHandler } from './$types'; @@ -58,7 +58,7 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { let content = ''; if (envFilePath && existsSync(envFilePath)) { try { - content = await Bun.file(envFilePath).text(); + content = readFileSync(envFilePath, 'utf-8'); } catch { // File read failed } @@ -154,7 +154,7 @@ export const PUT: RequestHandler = async ({ params, url, cookies, request }) => content += '\n'; } - await Bun.write(envFilePath, content); + writeFileSync(envFilePath, content); return json({ success: true }); } catch (error) { diff --git a/src/routes/api/stacks/[name]/restart/+server.ts b/src/routes/api/stacks/[name]/restart/+server.ts index b4d9ce3..90ccc6f 100644 --- a/src/routes/api/stacks/[name]/restart/+server.ts +++ b/src/routes/api/stacks/[name]/restart/+server.ts @@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit'; import { restartStack, ComposeFileNotFoundError } from '$lib/server/stacks'; import { authorize } from '$lib/server/authorize'; import { auditStack } from '$lib/server/audit'; +import { createJobResponse } from '$lib/server/sse'; import type { RequestHandler } from './$types'; export const POST: RequestHandler = async (event) => { @@ -21,22 +22,26 @@ export const POST: RequestHandler = async (event) => { return json({ error: 'Access denied to this environment' }, { status: 403 }); } - try { - const stackName = decodeURIComponent(params.name); - const result = await restartStack(stackName, envIdNum); + return createJobResponse(async (send) => { + try { + const stackName = decodeURIComponent(params.name); + const result = await restartStack(stackName, envIdNum); - // Audit log - await auditStack(event, 'restart', stackName, envIdNum); + // Audit log + await auditStack(event, 'restart', stackName, envIdNum); - if (!result.success) { - return json({ success: false, error: result.error }, { status: 400 }); + if (!result.success) { + send('result', { success: false, error: result.error }); + return; + } + send('result', { success: true, output: result.output }); + } catch (error) { + if (error instanceof ComposeFileNotFoundError) { + send('result', { success: false, error: error.message }); + return; + } + console.error('Error restarting compose stack:', error); + send('result', { success: false, error: 'Failed to restart compose stack' }); } - return json({ success: true, output: result.output }); - } catch (error) { - if (error instanceof ComposeFileNotFoundError) { - return json({ error: error.message }, { status: 404 }); - } - console.error('Error restarting compose stack:', error); - return json({ error: 'Failed to restart compose stack' }, { status: 500 }); - } + }, event.request); }; diff --git a/src/routes/api/stacks/[name]/start/+server.ts b/src/routes/api/stacks/[name]/start/+server.ts index 928841e..de52be3 100644 --- a/src/routes/api/stacks/[name]/start/+server.ts +++ b/src/routes/api/stacks/[name]/start/+server.ts @@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit'; import { startStack, ComposeFileNotFoundError } from '$lib/server/stacks'; import { authorize } from '$lib/server/authorize'; import { auditStack } from '$lib/server/audit'; +import { createJobResponse } from '$lib/server/sse'; import type { RequestHandler } from './$types'; export const POST: RequestHandler = async (event) => { @@ -21,22 +22,26 @@ export const POST: RequestHandler = async (event) => { return json({ error: 'Access denied to this environment' }, { status: 403 }); } - try { - const stackName = decodeURIComponent(params.name); - const result = await startStack(stackName, envIdNum); + return createJobResponse(async (send) => { + try { + const stackName = decodeURIComponent(params.name); + const result = await startStack(stackName, envIdNum); - // Audit log - await auditStack(event, 'start', stackName, envIdNum); + // Audit log + await auditStack(event, 'start', stackName, envIdNum); - if (!result.success) { - return json({ success: false, error: result.error }, { status: 400 }); + if (!result.success) { + send('result', { success: false, error: result.error }); + return; + } + send('result', { success: true, output: result.output }); + } catch (error) { + if (error instanceof ComposeFileNotFoundError) { + send('result', { success: false, error: error.message }); + return; + } + console.error('Error starting compose stack:', error); + send('result', { success: false, error: 'Failed to start compose stack' }); } - return json({ success: true, output: result.output }); - } catch (error) { - if (error instanceof ComposeFileNotFoundError) { - return json({ error: error.message }, { status: 404 }); - } - console.error('Error starting compose stack:', error); - return json({ error: 'Failed to start compose stack' }, { status: 500 }); - } + }, event.request); }; diff --git a/src/routes/api/stacks/[name]/stop/+server.ts b/src/routes/api/stacks/[name]/stop/+server.ts index 2c2a0fe..264466f 100644 --- a/src/routes/api/stacks/[name]/stop/+server.ts +++ b/src/routes/api/stacks/[name]/stop/+server.ts @@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit'; import { stopStack, ComposeFileNotFoundError } from '$lib/server/stacks'; import { authorize } from '$lib/server/authorize'; import { auditStack } from '$lib/server/audit'; +import { createJobResponse } from '$lib/server/sse'; import type { RequestHandler } from './$types'; export const POST: RequestHandler = async (event) => { @@ -21,22 +22,26 @@ export const POST: RequestHandler = async (event) => { return json({ error: 'Access denied to this environment' }, { status: 403 }); } - try { - const stackName = decodeURIComponent(params.name); - const result = await stopStack(stackName, envIdNum); + return createJobResponse(async (send) => { + try { + const stackName = decodeURIComponent(params.name); + const result = await stopStack(stackName, envIdNum); - // Audit log - await auditStack(event, 'stop', stackName, envIdNum); + // Audit log + await auditStack(event, 'stop', stackName, envIdNum); - if (!result.success) { - return json({ success: false, error: result.error }, { status: 400 }); + if (!result.success) { + send('result', { success: false, error: result.error }); + return; + } + send('result', { success: true, output: result.output }); + } catch (error) { + if (error instanceof ComposeFileNotFoundError) { + send('result', { success: false, error: error.message }); + return; + } + console.error('Error stopping compose stack:', error); + send('result', { success: false, error: 'Failed to stop compose stack' }); } - return json({ success: true, output: result.output }); - } catch (error) { - if (error instanceof ComposeFileNotFoundError) { - return json({ error: error.message }, { status: 404 }); - } - console.error('Error stopping compose stack:', error); - return json({ error: 'Failed to stop compose stack' }, { status: 500 }); - } + }, event.request); }; diff --git a/src/routes/api/stacks/scan/+server.ts b/src/routes/api/stacks/scan/+server.ts index 4314c5f..0b55f79 100644 --- a/src/routes/api/stacks/scan/+server.ts +++ b/src/routes/api/stacks/scan/+server.ts @@ -23,6 +23,7 @@ export const POST: RequestHandler = async ({ request, cookies }) => { // Detect which stacks are already running on any environment const discoveredWithRunning = await detectRunningStacks(result.discovered); + discoveredWithRunning.sort((a, b) => a.name.localeCompare(b.name)); return json({ ...result, diff --git a/src/routes/api/system/+server.ts b/src/routes/api/system/+server.ts index 7c27e54..001f4e0 100644 --- a/src/routes/api/system/+server.ts +++ b/src/routes/api/system/+server.ts @@ -13,6 +13,7 @@ import { isPostgres, isSqlite, getDatabaseSchemaVersion, getPostgresConnectionIn import { hasEnvironments } from '$lib/server/db'; import type { RequestHandler } from './$types'; import { existsSync, readFileSync } from 'node:fs'; +import * as http from 'node:http'; import os from 'node:os'; import { authorize } from '$lib/server/authorize'; @@ -47,12 +48,12 @@ function detectContainerRuntime(): { inContainer: boolean; runtime?: string; con return { inContainer: false }; } -// Get Bun runtime info -function getBunInfo() { +// Get runtime info +function getRuntimeInfo() { const memUsage = process.memoryUsage(); return { - version: typeof Bun !== 'undefined' ? Bun.version : null, - revision: typeof Bun !== 'undefined' ? Bun.revision?.slice(0, 7) : null, + name: 'Node.js', + version: process.version, memory: { heapUsed: memUsage.heapUsed, heapTotal: memUsage.heapTotal, @@ -67,7 +68,6 @@ async function getOwnContainerInfo(containerId: string | undefined): Promise((resolve, reject) => { + const req = http.request({ + socketPath, + path: `/containers/${containerId}/json`, + method: 'GET', + }, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => { + if (res.statusCode === 200) { + resolve(JSON.parse(Buffer.concat(chunks).toString('utf-8'))); + } else { + resolve(null); + } + }); + res.on('error', () => resolve(null)); + }); + req.on('error', () => resolve(null)); + req.end(); }); - if (response.ok) { - const info = await response.json(); + if (info) { return { id: info.Id?.slice(0, 12), name: info.Name?.replace(/^\//, ''), image: info.Config?.Image, - imageId: info.Image?.slice(7, 19), // Remove 'sha256:' prefix + imageId: info.Image?.slice(7, 19), created: info.Created, status: info.State?.Status, restartCount: info.RestartCount, @@ -153,7 +168,7 @@ export const GET: RequestHandler = async ({ url, cookies }) => { const runningContainers = containers.filter(c => c.state === 'running').length; const stoppedContainers = containers.length - runningContainers; - const bunInfo = getBunInfo(); + const runtimeInfo = getRuntimeInfo(); const containerRuntime = detectContainerRuntime(); const ownContainer = containerRuntime.inContainer ? await getOwnContainerInfo(containerRuntime.containerId || os.hostname()) @@ -181,13 +196,13 @@ export const GET: RequestHandler = async ({ url, cookies }) => { storageDriver: dockerInfo.Driver } : null, runtime: { - bun: bunInfo.version, - bunRevision: bunInfo.revision, - nodeVersion: process.version, + runtimeName: runtimeInfo.name, + runtimeVersion: runtimeInfo.version, + nodeVersion: runtimeInfo.version, platform: os.platform(), arch: os.arch(), kernel: os.release(), - memory: bunInfo.memory, + memory: runtimeInfo.memory, container: containerRuntime, ownContainer }, diff --git a/src/routes/api/users/+server.ts b/src/routes/api/users/+server.ts index 41f1b2b..259e50c 100644 --- a/src/routes/api/users/+server.ts +++ b/src/routes/api/users/+server.ts @@ -114,7 +114,7 @@ export const POST: RequestHandler = async (event) => { // Auto-login if this is the first user being created (and auth is enabled) let autoLoggedIn = false; if (isFirstUser && auth.authEnabled) { - await createUserSession(user.id, 'local', cookies); + await createUserSession(user.id, 'local', cookies, event.request); autoLoggedIn = true; } @@ -139,7 +139,7 @@ export const POST: RequestHandler = async (event) => { name: error.name, stack: error.stack }); - if (error.message?.includes('UNIQUE constraint failed') || error.code === '23505') { + if (error.message?.includes('UNIQUE constraint failed') || error.code === '23505' || (error as any).cause?.code === '23505') { return json({ error: 'Username already exists' }, { status: 409 }); } return json({ error: 'Failed to create user', details: error.message }, { status: 500 }); diff --git a/src/routes/api/users/[id]/+server.ts b/src/routes/api/users/[id]/+server.ts index d11aeb5..19d7c3a 100644 --- a/src/routes/api/users/[id]/+server.ts +++ b/src/routes/api/users/[id]/+server.ts @@ -229,7 +229,7 @@ export const PUT: RequestHandler = async (event) => { }); } catch (error: any) { console.error('Failed to update user:', error); - if (error.message?.includes('UNIQUE constraint failed')) { + if (error.message?.includes('UNIQUE constraint failed') || (error as any).cause?.code === '23505') { return json({ error: 'Username already exists' }, { status: 409 }); } return json({ error: 'Failed to update user' }, { status: 500 }); diff --git a/src/routes/api/volumes/+server.ts b/src/routes/api/volumes/+server.ts index 12366fd..4c45b87 100644 --- a/src/routes/api/volumes/+server.ts +++ b/src/routes/api/volumes/+server.ts @@ -1,6 +1,6 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -import { listVolumes, createVolume, EnvironmentNotFoundError, type CreateVolumeOptions } from '$lib/server/docker'; +import { listVolumes, createVolume, EnvironmentNotFoundError, DockerConnectionError, type CreateVolumeOptions } from '$lib/server/docker'; import { authorize } from '$lib/server/authorize'; import { auditVolume } from '$lib/server/audit'; import { hasEnvironments } from '$lib/server/db'; @@ -33,7 +33,9 @@ export const GET: RequestHandler = async ({ url, cookies }) => { if (error instanceof EnvironmentNotFoundError) { return json({ error: 'Environment not found' }, { status: 404 }); } - console.error('Failed to list volumes:', error); + if (!(error instanceof DockerConnectionError)) { + console.error('Failed to list volumes:', error); + } return json({ error: 'Failed to list volumes' }, { status: 500 }); } }; diff --git a/src/routes/api/volumes/[name]/export/+server.ts b/src/routes/api/volumes/[name]/export/+server.ts index 978edcb..7c68160 100644 --- a/src/routes/api/volumes/[name]/export/+server.ts +++ b/src/routes/api/volumes/[name]/export/+server.ts @@ -1,3 +1,4 @@ +import { gzipSync } from 'node:zlib'; import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import { getVolumeArchive } from '$lib/server/docker'; @@ -31,9 +32,9 @@ export const GET: RequestHandler = async ({ params, url, cookies }) => { let body: ReadableStream | Uint8Array = response.body!; if (format === 'tar.gz') { - // Compress with gzip using Bun's native implementation + // Compress with gzip const tarData = new Uint8Array(await response.arrayBuffer()); - body = Bun.gzipSync(tarData); + body = gzipSync(tarData); contentType = 'application/gzip'; extension = '.tar.gz'; } diff --git a/src/routes/containers/+page.svelte b/src/routes/containers/+page.svelte index 7c766f5..24ed5e0 100644 --- a/src/routes/containers/+page.svelte +++ b/src/routes/containers/+page.svelte @@ -73,9 +73,10 @@ import FileBrowserModal from './FileBrowserModal.svelte'; import BatchUpdateModal from './BatchUpdateModal.svelte'; import BatchOperationModal from '$lib/components/BatchOperationModal.svelte'; - import type { ContainerInfo, ContainerStats } from '$lib/types'; + import type { ContainerInfo } from '$lib/types'; import { EmptyState, NoEnvironment } from '$lib/components/ui/empty-state'; import { currentEnvironment, environments, appendEnvParam, clearStaleEnvironment } from '$lib/stores/environment'; + import { containerStore } from '$lib/stores/containers'; import { onDockerEvent, isContainerListChange } from '$lib/stores/events'; import { appSettings } from '$lib/stores/settings'; import { canAccess } from '$lib/stores/auth'; @@ -87,8 +88,7 @@ import type { ColumnConfig } from '$lib/types'; import type { DataGridRowState } from '$lib/components/data-grid/types'; - // Track previous stats for change detection - let previousStats = $state>(new Map()); + // Track change detection for stat highlighting (UI-only, stays in component) let changedFields = $state>>(new Map()); // Format bytes to human readable @@ -100,21 +100,26 @@ return parseFloat((bytes / Math.pow(k, i)).toFixed(decimals)) + sizes[i]; } - type SortField = 'name' | 'image' | 'state' | 'health' | 'uptime' | 'stack' | 'ip' | 'cpu' | 'memory'; + type SortField = 'name' | 'image' | 'state' | 'health' | 'uptime' | 'stack' | 'ip' | 'cpu' | 'memory' | 'ports'; type SortDirection = 'asc' | 'desc'; - let containers = $state([]); - let containerStats = $state>(new Map()); - let autoUpdateSettings = $state>(new Map()); + // Data from persistent store (survives page navigation) + const containers = $derived($containerStore.data); + const containerStats = $derived($containerStore.stats); + const autoUpdateSettings = $derived($containerStore.autoUpdateSettings); + const envHasScanning = $derived($containerStore.envHasScanning); + const envVulnerabilityCriteria = $derived($containerStore.envVulnerabilityCriteria); + const loading = $derived($containerStore.loading); + let envId = $state(null); - let envHasScanning = $state(false); - let envVulnerabilityCriteria = $state<'never' | 'any' | 'critical_high' | 'critical' | 'more_than_current'>('never'); // Derived: current environment details for reactive port URL generation const currentEnvDetails = $derived($environments.find(e => e.id === $currentEnvironment?.id) ?? null); - // Search and sort state - let searchQuery = $state(''); + // Search and sort state - initialize from URL for persistence across navigation + const initialSearch = $page.url.searchParams.get('search') + ?? $page.url.searchParams.get('image') ?? ''; + let searchQuery = $state(initialSearch); let sortField = $state('name'); let sortDirection = $state('asc'); @@ -167,6 +172,18 @@ saveStatusFilter(); }); + // Sync search query to URL for persistence across navigation + $effect(() => { + const q = searchQuery; + const url = new URL($page.url); + if (q) url.searchParams.set('search', q); + else url.searchParams.delete('search'); + url.searchParams.delete('image'); // clean up legacy param + if (url.toString() !== $page.url.toString()) { + goto(url.toString(), { replaceState: true, noScroll: true, keepFocus: true }); + } + }); + // Track if initial fetch has been done let initialFetchDone = $state(false); @@ -177,29 +194,28 @@ // Only fetch if environment actually changed or this is initial load if (env && (newEnvId !== envId || !initialFetchDone)) { + const isEnvSwitch = envId !== null && newEnvId !== envId; envId = newEnvId; initialFetchDone = true; // Clear update state from previous environment - batchUpdateContainerIds = []; - batchUpdateContainerNames = new Map(); updateCheckStatus = 'idle'; // Clear shell detection cache for new environment shellDetectionCache = {}; - fetchContainers(); - fetchStats(); - loadPendingUpdates(); + + if (isEnvSwitch) { + // Full env switch — invalidate cache, show spinner + containerStore.invalidate(); + } + // Refresh data (store handles loading state internally) + containerStore.refresh(newEnvId); } else if (!env) { // No environment - clear data and stop loading envId = null; - containers = []; - loading = false; - batchUpdateContainerIds = []; - batchUpdateContainerNames = new Map(); updateCheckStatus = 'idle'; shellDetectionCache = {}; + containerStore.clear(); } }); - let loading = $state(true); let showCreateModal = $state(false); let showEditModal = $state(false); let editContainerId = $state(''); @@ -247,8 +263,8 @@ // Update check state let updateCheckStatus = $state<'idle' | 'checking' | 'found' | 'none' | 'error'>('idle'); let showBatchUpdateModal = $state(false); - let batchUpdateContainerIds = $state([]); - let batchUpdateContainerNames = $state>(new Map()); + const batchUpdateContainerIds = $derived($containerStore.pendingUpdateIds); + const batchUpdateContainerNames = $derived($containerStore.pendingUpdateNames); // Single container update mode (doesn't overwrite batch list) let singleUpdateContainerId = $state(null); @@ -332,7 +348,7 @@ function handleBatchOpComplete() { selectedContainers = new Set(); - fetchContainers(); + containerStore.refreshContainers(envId); } function bulkStart() { @@ -353,9 +369,9 @@ function bulkRestart() { startBatchOperation( - `Restarting ${selectedInFilter.length} container${selectedInFilter.length !== 1 ? 's' : ''}`, + `Restarting ${selectedNonSystem.length} container${selectedNonSystem.length !== 1 ? 's' : ''}`, 'restart', - selectedInFilter + selectedNonSystem ); } @@ -377,9 +393,9 @@ function bulkRemove() { startBatchOperation( - `Removing ${selectedInFilter.length} container${selectedInFilter.length !== 1 ? 's' : ''}`, + `Removing ${selectedNonSystem.length} container${selectedNonSystem.length !== 1 ? 's' : ''}`, 'remove', - selectedInFilter, + selectedNonSystem, { force: true } ); } @@ -393,7 +409,7 @@ }); if (response.ok) { pruneStatus = 'success'; - await fetchContainers(); + await containerStore.refreshContainers(envId); } else { pruneStatus = 'error'; } @@ -418,23 +434,28 @@ } const data = await response.json(); const containersWithUpdates = data.results.filter((r: any) => r.hasUpdate); + const failedChecks = data.results.filter((r: any) => r.error && !r.hasUpdate).length; + const failedSuffix = failedChecks > 0 ? ` (${failedChecks} failed to check)` : ''; if (containersWithUpdates.length === 0) { updateCheckStatus = 'none'; - batchUpdateContainerIds = []; - batchUpdateContainerNames.clear(); - toast.success('All containers are up to date'); + containerStore.setPendingUpdates([], new Map()); + if (failedChecks > 0) { + toast.warning(`All containers are up to date${failedSuffix}`); + } else { + toast.success('All containers are up to date'); + } pendingTimeouts.push(setTimeout(() => { updateCheckStatus = 'idle'; }, 3000)); return; } // Prepare data for batch update modal (but don't open it yet) - batchUpdateContainerIds = containersWithUpdates.map((r: any) => r.containerId); - batchUpdateContainerNames = new Map( - containersWithUpdates.map((r: any) => [r.containerId, r.containerName]) + containerStore.setPendingUpdates( + containersWithUpdates.map((r: any) => r.containerId), + new Map(containersWithUpdates.map((r: any) => [r.containerId, r.containerName])) ); updateCheckStatus = 'found'; - toast.info(`${containersWithUpdates.length} update(s) available`); + toast.info(`${containersWithUpdates.length} update(s) available${failedSuffix}`); } catch (error) { updateCheckStatus = 'error'; pendingTimeouts.push(setTimeout(() => { updateCheckStatus = 'idle'; }, 3000)); @@ -444,19 +465,10 @@ // Load pending updates from database (persisted from check-updates or scheduled jobs) async function loadPendingUpdates() { if (!envId) return; - try { - const response = await fetch(appendEnvParam('/api/containers/pending-updates', envId)); - if (!response.ok) return; - const data = await response.json(); - if (data.pendingUpdates && data.pendingUpdates.length > 0) { - batchUpdateContainerIds = data.pendingUpdates.map((u: any) => u.containerId); - batchUpdateContainerNames = new Map( - data.pendingUpdates.map((u: any) => [u.containerId, u.containerName]) - ); - updateCheckStatus = 'found'; - } - } catch { - // Ignore errors - this is a background load + await containerStore.loadPendingUpdates(envId); + // Update local UI status if there are pending updates + if ($containerStore.pendingUpdateIds.length > 0) { + updateCheckStatus = 'found'; } } @@ -475,8 +487,7 @@ selectedNames.set(id, container.name); } } - batchUpdateContainerIds = selectedWithUpdates; - batchUpdateContainerNames = selectedNames; + containerStore.setPendingUpdates(selectedWithUpdates, selectedNames); showBatchUpdateModal = true; } @@ -492,7 +503,7 @@ } } if (allNames.size === 0) return; - batchUpdateContainerNames = allNames; + containerStore.patch({ pendingUpdateNames: allNames }); showBatchUpdateModal = true; } @@ -529,7 +540,7 @@ // Reload pending updates from database to restore highlighting for remaining containers loadPendingUpdates(); - fetchContainers(); + containerStore.refreshContainers(envId); } // Action in progress state (for animations) @@ -717,6 +728,15 @@ const stackB = b.labels?.['com.docker.compose.project'] || ''; cmp = stackA.localeCompare(stackB); break; + case 'ports': + const getLowestPort = (c: ContainerInfo) => { + const publicPorts = (c.ports || []) + .filter((p: any) => p.PublicPort) + .map((p: any) => p.PublicPort!); + return publicPorts.length > 0 ? Math.min(...publicPorts) : Infinity; + }; + cmp = getLowestPort(a) - getLowestPort(b); + break; case 'ip': const ipA = getContainerIp(a.networks); const ipB = getContainerIp(b.networks); @@ -758,122 +778,15 @@ filteredContainers.filter(c => selectedContainers.has(c.id)) ); - // Count by state for selected containers - const selectedRunning = $derived(selectedInFilter.filter(c => c.state === 'running')); - const selectedStopped = $derived(selectedInFilter.filter(c => c.state !== 'running' && c.state !== 'paused')); - const selectedPaused = $derived(selectedInFilter.filter(c => c.state === 'paused')); + // Count by state for selected containers (exclude system containers from destructive actions) + const selectedNonSystem = $derived(selectedInFilter.filter(c => !c.systemContainer)); + const selectedRunning = $derived(selectedNonSystem.filter(c => c.state === 'running')); + const selectedStopped = $derived(selectedNonSystem.filter(c => c.state !== 'running' && c.state !== 'paused')); + const selectedPaused = $derived(selectedNonSystem.filter(c => c.state === 'paused')); - async function fetchContainers() { - loading = true; - try { - const response = await fetch(appendEnvParam('/api/containers', envId)); - if (!response.ok) { - // Handle stale environment ID (e.g., after database reset) - if (response.status === 404 && envId) { - clearStaleEnvironment(envId); - environments.refresh(); - return; - } - toast.error('Failed to load containers'); - return; - } - containers = await response.json(); - // Fetch auto-update settings for all containers - await fetchAutoUpdateSettings(); - } catch (error) { - console.error('Failed to fetch containers:', error); - toast.error('Failed to load containers'); - } finally { - loading = false; - } - } - - async function checkScannerSettings() { - if (!envId) { - envHasScanning = false; - envVulnerabilityCriteria = 'never'; - return; - } - try { - // Fetch scanner settings and environment update-check settings in parallel - const [scannerResponse, updateCheckResponse] = await Promise.all([ - fetch(`/api/settings/scanner?env=${envId}&settingsOnly=true`), - fetch(`/api/environments/${envId}/update-check`) - ]); - - if (scannerResponse.ok) { - const data = await scannerResponse.json(); - const settings = data.settings || data; - envHasScanning = settings.scanner !== 'none'; - } - - if (updateCheckResponse.ok) { - const data = await updateCheckResponse.json(); - envVulnerabilityCriteria = data.settings?.vulnerabilityCriteria || 'never'; - } - } catch { - envHasScanning = false; - envVulnerabilityCriteria = 'never'; - } - } - - async function fetchAutoUpdateSettings() { - const settings = new Map(); - const envParam = envId ? `?env=${envId}` : ''; - - // Check scanner settings first - await checkScannerSettings(); - - // Fetch all auto-update settings in a single request - try { - const response = await fetch(`/api/auto-update${envParam}`); - if (response.ok) { - const data = await response.json(); - // data is a map of containerName -> settings - for (const [containerName, setting] of Object.entries(data)) { - if (setting && typeof setting === 'object' && 'enabled' in setting && setting.enabled) { - const s = setting as { enabled: boolean; scheduleType: string; cronExpression: string | null; vulnerabilityCriteria: string }; - const { label, tooltip } = formatSchedule(s.scheduleType, s.cronExpression || ''); - settings.set(containerName, { - enabled: true, - label, - tooltip, - vulnerabilityCriteria: s.vulnerabilityCriteria || 'never' - }); - } - } - } - } catch (err) { - console.error('Failed to fetch auto-update settings:', err); - } - - autoUpdateSettings = settings; - } - - function formatSchedule(scheduleType: string, cronExpression: string): { label: string; tooltip: string } { - if (!cronExpression) return { label: 'on', tooltip: 'Auto-update enabled' }; - - const parts = cronExpression.split(' '); - if (parts.length < 5) return { label: 'cron', tooltip: cronExpression }; - - const [min, hr, , , dow] = parts; - const hourNum = parseInt(hr); - const minNum = parseInt(min); - const ampm = hourNum >= 12 ? 'PM' : 'AM'; - const hour12 = hourNum === 0 ? 12 : hourNum > 12 ? hourNum - 12 : hourNum; - const timeStr = `${hour12}:${minNum.toString().padStart(2, '0')} ${ampm}`; - - if (scheduleType === 'daily' || dow === '*') { - return { label: 'daily', tooltip: `Daily at ${timeStr}` }; - } - - if (scheduleType === 'weekly') { - const days = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']; - const dayName = days[parseInt(dow)] || dow; - return { label: dayName, tooltip: `Every ${['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][parseInt(dow)] || dow} at ${timeStr}` }; - } - - return { label: 'cron', tooltip: cronExpression }; + // Thin wrappers — delegate to persistent store + function fetchContainers() { + return containerStore.refreshContainers(envId); } // Check if highlightChanges is enabled for current environment @@ -896,49 +809,37 @@ return changedFields.get(containerId)?.has(field) ?? false; } - async function fetchStats() { - try { - const response = await fetch(appendEnvParam('/api/containers/stats', envId)); - const stats: ContainerStats[] = await response.json(); - const statsMap = new Map(); - const newChangedFields = new Map>(); + // Detect stat changes for highlighting when store stats update + $effect(() => { + const currentStats = $containerStore.stats; + const prevStats = $containerStore.previousStats; - for (const stat of stats) { - statsMap.set(stat.id, stat); + if (!highlightChangesEnabled || prevStats.size === 0) return; - // Track changes if highlighting is enabled - if (highlightChangesEnabled) { - const prev = previousStats.get(stat.id); - if (prev) { - const changes = new Set(); - if (hasFieldChanged(stat.id, 'cpu', prev.cpuPercent, stat.cpuPercent)) changes.add('cpu'); - if (hasFieldChanged(stat.id, 'memory', prev.memoryUsage, stat.memoryUsage)) changes.add('memory'); - if (hasFieldChanged(stat.id, 'networkRx', prev.networkRx, stat.networkRx)) changes.add('network'); - if (hasFieldChanged(stat.id, 'networkTx', prev.networkTx, stat.networkTx)) changes.add('network'); - if (hasFieldChanged(stat.id, 'blockRead', prev.blockRead, stat.blockRead)) changes.add('disk'); - if (hasFieldChanged(stat.id, 'blockWrite', prev.blockWrite, stat.blockWrite)) changes.add('disk'); - if (changes.size > 0) { - newChangedFields.set(stat.id, changes); - } - } + const newChangedFields = new Map>(); + for (const [id, stat] of currentStats) { + const prev = prevStats.get(id); + if (prev) { + const changes = new Set(); + if (hasFieldChanged(id, 'cpu', prev.cpuPercent, stat.cpuPercent)) changes.add('cpu'); + if (hasFieldChanged(id, 'memory', prev.memoryUsage, stat.memoryUsage)) changes.add('memory'); + if (hasFieldChanged(id, 'networkRx', prev.networkRx, stat.networkRx)) changes.add('network'); + if (hasFieldChanged(id, 'networkTx', prev.networkTx, stat.networkTx)) changes.add('network'); + if (hasFieldChanged(id, 'blockRead', prev.blockRead, stat.blockRead)) changes.add('disk'); + if (hasFieldChanged(id, 'blockWrite', prev.blockWrite, stat.blockWrite)) changes.add('disk'); + if (changes.size > 0) { + newChangedFields.set(id, changes); } } - - // Update changed fields and clear after animation duration - changedFields = newChangedFields; - if (newChangedFields.size > 0) { - pendingTimeouts.push(setTimeout(() => { - changedFields = new Map(); - }, 1500)); - } - - // Store current stats as previous for next comparison - previousStats = new Map(statsMap); - containerStats = statsMap; - } catch (error) { - console.error('Failed to fetch container stats:', error); } - } + + changedFields = newChangedFields; + if (newChangedFields.size > 0) { + pendingTimeouts.push(setTimeout(() => { + changedFields = new Map(); + }, 1500)); + } + }); async function startContainer(id: string) { operationError = null; @@ -954,7 +855,7 @@ return; } toast.success(`Started ${name}`); - await fetchContainers(); + await containerStore.refreshContainers(envId); } catch (error) { console.error('Failed to start container:', error); operationError = { id, message: 'Failed to start container' }; @@ -978,7 +879,7 @@ return; } toast.success(`Stopped ${name}`); - await fetchContainers(); + await containerStore.refreshContainers(envId); } catch (error) { console.error('Failed to stop container:', error); operationError = { id, message: 'Failed to stop container' }; @@ -1003,7 +904,7 @@ return; } toast.success(`Paused ${name}`); - await fetchContainers(); + await containerStore.refreshContainers(envId); } catch (error) { console.error('Failed to pause container:', error); operationError = { id, message: 'Failed to pause container' }; @@ -1026,7 +927,7 @@ return; } toast.success(`Resumed ${name}`); - await fetchContainers(); + await containerStore.refreshContainers(envId); } catch (error) { console.error('Failed to unpause container:', error); operationError = { id, message: 'Failed to unpause container' }; @@ -1050,7 +951,7 @@ return; } toast.success(`Restarted ${name}`); - await fetchContainers(); + await containerStore.refreshContainers(envId); } catch (error) { console.error('Failed to restart container:', error); operationError = { id, message: 'Failed to restart container' }; @@ -1075,7 +976,7 @@ return; } toast.success(`Removed ${name}`); - await fetchContainers(); + await containerStore.refreshContainers(envId); } catch (error) { console.error('Failed to remove container:', error); operationError = { id, message: 'Failed to remove container' }; @@ -1379,8 +1280,8 @@ function handleVisibilityChange() { if (document.visibilityState === 'visible' && envId) { // Tab became visible - refresh data immediately - fetchContainers(); - fetchStats(); + containerStore.refreshContainers(envId); + containerStore.refreshStats(envId); } } @@ -1388,12 +1289,6 @@ loadLayoutMode(); loadStatusFilter(); - // Check for image filter from URL params (from images page link) - const imageParam = $page.url.searchParams.get('image'); - if (imageParam) { - searchQuery = imageParam; - } - // Load persisted pending updates from database loadPendingUpdates(); @@ -1405,14 +1300,14 @@ // Set up interval to refresh stats every 5 seconds (use module-scope var for cleanup) statsInterval = setInterval(() => { - if (envId) fetchStats(); + if (envId) containerStore.refreshStats(envId); }, 5000); // Subscribe to container events (SSE connection is global in layout) unsubscribeDockerEvent = onDockerEvent((event) => { if (envId && isContainerListChange(event)) { - fetchContainers(); - fetchStats(); + containerStore.refreshContainers(envId); + containerStore.refreshStats(envId); } }); @@ -1634,12 +1529,12 @@ {/snippet} {/if} - {#if $canAccess('containers', 'restart')} + {#if selectedNonSystem.length > 0 && $canAccess('containers', 'restart')} {/if} - {#if $canAccess('containers', 'remove')} + {#if selectedNonSystem.length > 0 && $canAccess('containers', 'remove')} confirmBulkRemove = open} @@ -1897,7 +1792,7 @@ {:else if column.id === 'ports'} {#if ports.length > 0}

- {#each ports.slice(0, 2) as port} + {#each ports as port} {@const url = currentEnvDetails ? getPortUrl(port.publicPort) : null} {#if url} {port.display} {/if} {/each} - {#if ports.length > 2} - +{ports.length - 2} - {/if}
{:else} - @@ -1943,13 +1835,20 @@ {/if} {:else if column.id === 'stack'} {#if stack} - + + + + + +

{stack}

+ + {:else} - {/if} @@ -1965,6 +1864,7 @@ {/if} + {#if !container.systemContainer} {#if container.state === 'running' || container.state === 'restarting'} {#if $canAccess('containers', 'stop')} {/if} + {/if} + {#if $canAccess('stacks', 'create')} + {#if container.state === 'running' && $canAccess('containers', 'exec')} - + {:else if steps.length === 0 && isDeploying} +
+ + Initializing...
- - {:else} - -
-
- - {stackName} + {:else} +
+ {#each steps as step, index (index)} + {@const StepIcon = getStepIcon(step.status)} + {@const isCurrentStep = index === steps.length - 1 && isDeploying} +
+ + + {step.message || step.status} + +
+ {/each}
+ {/if} - -
-
- {#if overallStatus === 'idle'} - - Initializing... - {:else if overallStatus === 'deploying'} - - Deploying... - {:else if overallStatus === 'complete'} - - Complete! - {:else if overallStatus === 'error'} - - Failed - {/if} -
- {#if currentStep?.step && currentStep?.totalSteps} - - {currentStep.step}/{currentStep.totalSteps} - - {/if} -
- - {#if currentStep?.message && overallStatus === 'deploying'} -

{currentStep.message}

- {/if} - - {#if currentStep?.totalSteps} - - {/if} - - {#if errorMessage} -
- + {#if errorMessage} +
+
+ {errorMessage}
+
+ {/if} +
+ + +
+ +
+ {#if overallStatus === 'confirming'} + + {:else if steps.length > 0} + {/if}
- - {#if steps.length > 0} -
-
- {#each steps as step, index (index)} - {@const StepIcon = getStepIcon(step.status)} - {@const isCurrentStep = index === steps.length - 1 && overallStatus === 'deploying'} -
- - - {step.message || step.status} - -
- {/each} -
-
- {/if} - - - {#if overallStatus === 'complete' || overallStatus === 'error'} -
- -
- {/if} - {/if} - - + {:else} + + {/if} +
+
+ + diff --git a/src/routes/stacks/GitStackModal.svelte b/src/routes/stacks/GitStackModal.svelte index abd594d..c967259 100644 --- a/src/routes/stacks/GitStackModal.svelte +++ b/src/routes/stacks/GitStackModal.svelte @@ -14,6 +14,7 @@ import { type EnvVar, type ValidationResult } from '$lib/components/StackEnvVarsEditor.svelte'; import { toast } from 'svelte-sonner'; import { focusFirstInput } from '$lib/utils'; + import { readJobResponse } from '$lib/utils/sse-fetch'; import { useSidebar } from '$lib/components/ui/sidebar/context.svelte'; // Get sidebar state to adjust modal positioning @@ -465,7 +466,7 @@ body: JSON.stringify(body) }); - const data = await response.json(); + const data = await readJobResponse(response); if (!response.ok) { formError = data.error || 'Failed to save git stack'; diff --git a/src/routes/stacks/ImportStackModal.svelte b/src/routes/stacks/ImportStackModal.svelte index 03fd1e4..bd09fcc 100644 --- a/src/routes/stacks/ImportStackModal.svelte +++ b/src/routes/stacks/ImportStackModal.svelte @@ -150,9 +150,14 @@ } scanResults = discovered; + const skippedCount = (data.skipped || []).length; if (discovered.length === 0) { - toast.info('No compose stacks found in this directory'); + if (skippedCount > 0) { + toast.info(`All ${skippedCount} stack(s) in this directory are already adopted`); + } else { + toast.info('No compose stacks found in this directory'); + } } else { const selections = new Map(); for (const stack of discovered) { diff --git a/src/routes/stacks/StackModal.svelte b/src/routes/stacks/StackModal.svelte index ed27b00..19d15a1 100644 --- a/src/routes/stacks/StackModal.svelte +++ b/src/routes/stacks/StackModal.svelte @@ -20,6 +20,8 @@ import { copyToClipboard } from '$lib/utils/clipboard'; import * as Alert from '$lib/components/ui/alert'; import { ErrorDialog } from '$lib/components/ui/error-dialog'; + import { readJobResponse } from '$lib/utils/sse-fetch'; + import { toast } from 'svelte-sonner'; import ComposeGraphViewer from './ComposeGraphViewer.svelte'; import { useSidebar } from '$lib/components/ui/sidebar/context.svelte'; @@ -37,7 +39,11 @@ onSuccess: () => void; // Called after create or save } - let { open = $bindable(), mode, stackName = '', onClose, onSuccess }: Props = $props(); + let { open = $bindable(), mode: propMode, stackName: propStackName = '', onClose, onSuccess }: Props = $props(); + + // Local effective state - can transition from create → edit after failed deploy + let mode = $state(propMode); + let stackName = $state(propStackName); // Form state let newStackName = $state(''); @@ -930,11 +936,17 @@ services: body: JSON.stringify(requestBody) }); - if (!response.ok) { - const data = await response.json(); + // When start=true, response is a job or JSON; when start=false, it's plain JSON + const data = start ? await readJobResponse(response) : await response.json(); + + if (!response.ok && !data.success) { throw new Error((typeof data.error === 'string' ? data.error : data.message) || 'Failed to create stack'); } + if (data.success === false) { + throw new Error(data.error || 'Failed to create stack'); + } + toast.success(`Created stack "${newStackName.trim()}"`); onSuccess(); handleClose(); } catch (e: any) { @@ -943,6 +955,13 @@ services: message: e.message || 'An error occurred while creating the stack', details: e.details }; + // If start=true, files were saved and stack is in DB — transition to edit mode + // so the user can fix and redeploy without leaving the modal + if (start) { + mode = 'edit'; + stackName = newStackName.trim(); + onSuccess(); // refresh stack list so the new stack appears + } } finally { saving = false; } @@ -1094,13 +1113,18 @@ services: } ); - const data = await response.json(); + // When restart=true, response is a job or JSON; when restart=false, it's plain JSON + const data = restart ? await readJobResponse(response) : await response.json(); - if (!response.ok) { + if (!response.ok && !data.success) { throw new Error((typeof data.error === 'string' ? data.error : data.message) || 'Failed to save compose file'); } + if (data.success === false) { + throw new Error(data.error || 'Failed to save compose file'); + } isDirty = false; // Reset dirty flag after successful save + toast.success(restart ? 'Stack applied' : 'Stack saved'); onSuccess(); if (!restart) { @@ -1147,6 +1171,9 @@ services: clearTimeout(validateTimer); validateTimer = null; } + // Reset mode back to prop values + mode = propMode; + stackName = propStackName; // Reset all state newStackName = ''; error = null; @@ -1196,6 +1223,9 @@ services: $effect(() => { if (open && !hasInitialized) { hasInitialized = true; + // Reset mode to prop values on each open + mode = propMode; + stackName = propStackName; if (mode === 'edit' && stackName) { loadComposeFile().then(() => { // Auto-validate after loading diff --git a/src/routes/terminal/+page.svelte b/src/routes/terminal/+page.svelte index b165b41..2ca1cf9 100644 --- a/src/routes/terminal/+page.svelte +++ b/src/routes/terminal/+page.svelte @@ -62,7 +62,7 @@ let connectedPollInterval: ReturnType | null = null; // Subscribe to environment changes - currentEnvironment.subscribe((env) => { + const unsubscribeEnv = currentEnvironment.subscribe((env) => { envId = env?.id ?? null; if (env) { fetchContainers(); @@ -201,6 +201,7 @@ }); onDestroy(() => { + unsubscribeEnv(); if (containerInterval) { clearInterval(containerInterval); containerInterval = null; diff --git a/svelte.config.js b/svelte.config.js index 54e37c6..87b4668 100644 --- a/svelte.config.js +++ b/svelte.config.js @@ -1,4 +1,4 @@ -import adapter from 'svelte-adapter-bun'; +import adapter from '@sveltejs/adapter-node'; import { vitePreprocess } from '@sveltejs/vite-plugin-svelte'; /** @type {import('@sveltejs/kit').Config} */ diff --git a/vite.config.ts b/vite.config.ts index 0be124e..40353ec 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -5,7 +5,13 @@ import { execSync } from 'child_process'; import { existsSync, readFileSync } from 'fs'; import { homedir } from 'os'; import { join } from 'path'; -import { Database } from 'bun:sqlite'; +import Database from 'better-sqlite3'; +import { WebSocketServer, WebSocket as WsWebSocket } from 'ws'; +import * as net from 'node:net'; +import * as tls from 'node:tls'; +import * as http from 'node:http'; +import * as https from 'node:https'; +import argon2 from 'argon2'; import { createDecipheriv } from 'node:crypto'; // ============ Encryption/Decryption for dev mode ============ @@ -228,7 +234,7 @@ function getGitCommit(): string | null { // Check COMMIT file (created by CI/CD before docker build) try { if (existsSync('COMMIT')) { - const commit = require('fs').readFileSync('COMMIT', 'utf-8').trim(); + const commit = readFileSync('COMMIT', 'utf-8').trim(); if (commit && commit !== 'unknown') { return commit; } @@ -248,7 +254,7 @@ function getGitBranch(): string | null { // Check BRANCH file (created by CI/CD before docker build) try { if (existsSync('BRANCH')) { - const branch = require('fs').readFileSync('BRANCH', 'utf-8').trim(); + const branch = readFileSync('BRANCH', 'utf-8').trim(); if (branch && branch !== 'unknown') { return branch; } @@ -272,7 +278,7 @@ function getGitTag(): string | null { // Check VERSION file (created by CI/CD before docker build) try { if (existsSync('VERSION')) { - const version = require('fs').readFileSync('VERSION', 'utf-8').trim(); + const version = readFileSync('VERSION', 'utf-8').trim(); if (version && version !== 'unknown') { return version; } @@ -288,20 +294,6 @@ function getGitTag(): string | null { } } -// Plugin to externalize bun: protocol modules -function bunExternals(): Plugin { - return { - name: 'bun-externals', - enforce: 'pre', - resolveId(source) { - if (source.startsWith('bun:')) { - return { id: source, external: true }; - } - return null; - } - }; -} - // Detect Docker socket path function detectDockerSocket(): string { if (process.env.DOCKER_SOCKET && existsSync(process.env.DOCKER_SOCKET)) return process.env.DOCKER_SOCKET; @@ -350,68 +342,55 @@ function getDockerTarget(envId?: number): DockerTarget { ); } +// Helper to make HTTP requests to Docker (supports Unix sockets and TCP with TLS) +function dockerHttpRequest(method: string, path: string, target: DockerTarget, body?: string): Promise<{ statusCode: number; body: string }> { + return new Promise((resolve, reject) => { + const headers: Record = {}; + if (body) headers['Content-Type'] = 'application/json'; + if (target.hawserToken) headers['X-Hawser-Token'] = target.hawserToken; + if (body) headers['Content-Length'] = Buffer.byteLength(body).toString(); + + const opts: any = { method, headers, path }; + + let req: any; + if (target.type === 'unix') { + opts.socketPath = target.socket; + req = http.request(opts); + } else if (target.tls) { + opts.host = target.host; + opts.port = target.port; + opts.rejectUnauthorized = target.tls.rejectUnauthorized ?? true; + if (target.tls.ca) opts.ca = [target.tls.ca]; + if (target.tls.cert) opts.cert = [target.tls.cert]; + if (target.tls.key) opts.key = target.tls.key; + req = https.request(opts); + } else { + opts.host = target.host; + opts.port = target.port; + req = http.request(opts); + } + + req.on('response', (res: any) => { + let data = ''; + res.on('data', (chunk: Buffer) => { data += chunk.toString(); }); + res.on('end', () => resolve({ statusCode: res.statusCode, body: data })); + }); + req.on('error', reject); + if (body) req.write(body); + req.end(); + }); +} + async function createExecForWs(containerId: string, cmd: string[], user: string, target: ReturnType): Promise<{ Id: string }> { - const headers: Record = { 'Content-Type': 'application/json' }; - const fetchOpts: any = { - method: 'POST', - headers, - body: JSON.stringify({ AttachStdin: true, AttachStdout: true, AttachStderr: true, Tty: true, Cmd: cmd, User: user }) - }; - let url: string; - if (target.type === 'unix') { - url = 'http://localhost/containers/' + containerId + '/exec'; - fetchOpts.unix = target.socket; - } else { - const protocol = target.tls ? 'https' : 'http'; - url = protocol + '://' + target.host + ':' + target.port + '/containers/' + containerId + '/exec'; - if (target.hawserToken) { - headers['X-Hawser-Token'] = target.hawserToken; - } - // Add TLS options for mTLS connections - if (target.tls) { - fetchOpts.tls = { - sessionTimeout: 0, // Disable TLS session caching - servername: target.host, - rejectUnauthorized: target.tls.rejectUnauthorized ?? true - }; - if (target.tls.ca) fetchOpts.tls.ca = [target.tls.ca]; - if (target.tls.cert) fetchOpts.tls.cert = [target.tls.cert]; - if (target.tls.key) fetchOpts.tls.key = target.tls.key; - fetchOpts.keepalive = false; - } - } - const res = await fetch(url, fetchOpts); - if (!res.ok) throw new Error('Failed to create exec: ' + (await res.text())); - return res.json(); + const body = JSON.stringify({ AttachStdin: true, AttachStdout: true, AttachStderr: true, Tty: true, Cmd: cmd, User: user }); + const res = await dockerHttpRequest('POST', '/containers/' + containerId + '/exec', target, body); + if (res.statusCode !== 201) throw new Error('Failed to create exec: ' + res.body); + return JSON.parse(res.body); } async function resizeExecForWs(execId: string, cols: number, rows: number, target: ReturnType): Promise { try { - const fetchOpts: any = { method: 'POST' }; - let url: string; - if (target.type === 'unix') { - url = 'http://localhost/exec/' + execId + '/resize?h=' + rows + '&w=' + cols; - fetchOpts.unix = target.socket; - } else { - const protocol = target.tls ? 'https' : 'http'; - url = protocol + '://' + target.host + ':' + target.port + '/exec/' + execId + '/resize?h=' + rows + '&w=' + cols; - if (target.hawserToken) { - fetchOpts.headers = { 'X-Hawser-Token': target.hawserToken }; - } - // Add TLS options for mTLS connections - if (target.tls) { - fetchOpts.tls = { - sessionTimeout: 0, - servername: target.host, - rejectUnauthorized: target.tls.rejectUnauthorized ?? true - }; - if (target.tls.ca) fetchOpts.tls.ca = [target.tls.ca]; - if (target.tls.cert) fetchOpts.tls.cert = [target.tls.cert]; - if (target.tls.key) fetchOpts.tls.key = target.tls.key; - fetchOpts.keepalive = false; - } - } - await fetch(url, fetchOpts); + await dockerHttpRequest('POST', '/exec/' + execId + '/resize?h=' + rows + '&w=' + cols, target); } catch { // Ignore resize errors } @@ -462,6 +441,20 @@ function startCleanupInterval() { if (dockerCleaned > 0 || edgeCleaned > 0) { console.log(`[WS Cleanup] Removed ${dockerCleaned} orphaned docker streams, ${edgeCleaned} orphaned edge sessions`); } + + // Maintain reconnection tracker: reset for stable connections, prune stale entries + const now = Date.now(); + for (const [envId, tracker] of reconnectTracker) { + const conn = edgeConnections.get(envId); + if (conn && now - conn.lastHeartbeat < STABLE_THRESHOLD_MS) { + reconnectTracker.delete(envId); + } else if (!conn && tracker.timestamps.length > 0) { + const lastAttempt = tracker.timestamps[tracker.timestamps.length - 1]; + if (now - lastAttempt > STALE_TRACKER_MS) { + reconnectTracker.delete(envId); + } + } + } }, 5 * 60 * 1000); } @@ -476,7 +469,7 @@ interface EdgeConnection { hostname: string; capabilities: string[]; connectedAt: Date; - lastHeartbeat: Date; + lastHeartbeat: number; pendingRequests: Map; pendingStreamRequests: Map; pingInterval?: ReturnType; // Server-side ping to keep connection alive through proxies @@ -543,332 +536,348 @@ function webSocketPlugin(): Plugin { // Start cleanup interval for dev mode only startCleanupInterval(); + // Start Hawser auth fail cache cleanup (dev mode only, not during build) + setInterval(() => { + const now = Date.now(); + for (const [key, ts] of hawserAuthFailCache) { + if (now - ts > HAWSER_AUTH_FAIL_COOLDOWN_MS) hawserAuthFailCache.delete(key); + } + }, 5 * 60_000); + const dockerSocketPath = detectDockerSocket(); console.log(`[Terminal WS] Detected Docker socket at: ${dockerSocketPath}`); - // Start a Bun.serve WebSocket server on a separate port - Bun.serve({ - port: WS_PORT, - fetch(req, server) { - // Upgrade HTTP requests to WebSocket - if (server.upgrade(req, { data: { url: req.url } })) { - return; // Return nothing if upgrade succeeds + // Start a ws WebSocket server on a separate port + const httpServer = http.createServer((_req: any, res: any) => { + res.writeHead(200); + res.end('WebSocket server'); + }); + + const wss = new WebSocketServer({ server: httpServer }); + + // Per-connection metadata + const wsMetadata = new Map(); + + wss.on('connection', (ws: WsWebSocket, req: any) => { + const url = new URL(req.url || '/', `http://localhost:${WS_PORT}`); + const meta = { url: req.url || '/' }; + wsMetadata.set(ws, meta); + + // Handle connection open logic + (async () => { + // Check if this is a Hawser Edge connection + if (url.pathname === '/api/hawser/connect') { + console.log('[Hawser WS] New connection pending authentication'); + return; } - return new Response('WebSocket server', { status: 200 }); - }, - websocket: { - async open(ws) { - const url = new URL((ws.data as any).url, `http://localhost:${WS_PORT}`); - // Check if this is a Hawser Edge connection - if (url.pathname === '/api/hawser/connect') { - console.log('[Hawser WS] New connection pending authentication'); - // Hawser connections wait for hello message to authenticate - return; - } + // Assign unique connection ID to this WebSocket + const connId = `ws-${++wsConnectionCounter}`; + meta.connId = connId; - // Assign unique connection ID to this WebSocket - const connId = `ws-${++wsConnectionCounter}`; - (ws.data as any).connId = connId; + // Terminal connection handling + const pathParts = url.pathname.split('/'); + const containerIdIndex = pathParts.indexOf('containers') + 1; + const containerId = pathParts[containerIdIndex]; - // Terminal connection handling - const pathParts = url.pathname.split('/'); - const containerIdIndex = pathParts.indexOf('containers') + 1; - const containerId = pathParts[containerIdIndex]; + const shell = url.searchParams.get('shell') || '/bin/sh'; + const user = url.searchParams.get('user') || 'root'; + const envIdParam = url.searchParams.get('envId'); + const envId = envIdParam ? parseInt(envIdParam, 10) : undefined; - const shell = url.searchParams.get('shell') || '/bin/sh'; - const user = url.searchParams.get('user') || 'root'; - const envIdParam = url.searchParams.get('envId'); - const envId = envIdParam ? parseInt(envIdParam, 10) : undefined; + if (!containerId) { + ws.send(JSON.stringify({ type: 'error', message: 'No container ID' })); + ws.close(); + return; + } - if (!containerId) { - ws.send(JSON.stringify({ type: 'error', message: 'No container ID' })); - ws.close(); - return; - } + const target = getDockerTarget(envId); - const target = getDockerTarget(envId); - - try { - // Handle Hawser Edge mode differently - use WebSocket protocol - if (target.type === 'hawser-edge') { - const conn = edgeConnections.get(target.environmentId); - if (!conn) { - ws.send(JSON.stringify({ type: 'error', message: 'Edge agent not connected' })); - ws.close(); - return; - } - - // Generate unique exec ID - const execId = crypto.randomUUID(); - - // Track this session - edgeExecSessions.set(execId, { ws, execId, environmentId: target.environmentId }); - (ws.data as any).edgeExecId = execId; - - // Send exec_start to the agent (using shared helper) - const execStartMsg = createExecStartMessage(execId, containerId, shell, user); - conn.ws.send(JSON.stringify(execStartMsg)); + try { + // Handle Hawser Edge mode differently - use WebSocket protocol + if (target.type === 'hawser-edge') { + const conn = edgeConnections.get(target.environmentId); + if (!conn) { + ws.send(JSON.stringify({ type: 'error', message: 'Edge agent not connected' })); + ws.close(); return; } - // Direct Docker connection (unix or tcp/hawser-standard) - const exec = await createExecForWs(containerId, [shell], user, target); - const execId = exec.Id; + const execId = crypto.randomUUID(); + edgeExecSessions.set(execId, { ws, execId, environmentId: target.environmentId }); + meta.edgeExecId = execId; - // Track connection state (using object for mutability across closures) - let headersStripped = false; - const state = { isChunked: false }; + const execStartMsg = createExecStartMessage(execId, containerId, shell, user); + conn.ws.send(JSON.stringify(execStartMsg)); + return; + } - // Create socket handler for Docker connection - const socketHandler = { - data(socket: any, data: Buffer) { - if (ws.readyState === 1) { - let text = new TextDecoder().decode(data); - // Skip HTTP headers in first response (only once) - if (!headersStripped) { - // Check for chunked encoding in headers - if (text.toLowerCase().includes('transfer-encoding: chunked')) { - state.isChunked = true; - } - const headerEnd = text.indexOf('\r\n\r\n'); - if (headerEnd > -1) { - text = text.slice(headerEnd + 4); - headersStripped = true; - } else if (text.startsWith('HTTP/')) { - // Headers split across packets, skip this entire packet - return; - } - } - // Strip chunked encoding framing if detected - if (state.isChunked && text) { - // Remove chunk size lines (hex number followed by \r\n) - text = text.replace(/^[0-9a-fA-F]+\r\n/gm, '').replace(/\r\n$/g, ''); - } - if (text) { - ws.send(JSON.stringify({ type: 'output', data: text })); - } - } - }, - close() { - if (ws.readyState === 1) { - ws.send(JSON.stringify({ type: 'exit' })); - ws.close(); - } - }, - error(socket: any, error: any) { - console.error('[Terminal WS] Socket error:', error?.message || error); - if (ws.readyState === 1) { - ws.send(JSON.stringify({ type: 'error', message: `Connection error: ${error?.message || 'Unknown error'}` })); - } - }, - connectError(socket: any, error: any) { - console.error('[Terminal WS] Connect error:', error?.message || error); - if (ws.readyState === 1) { - ws.send(JSON.stringify({ type: 'error', message: `Failed to connect: ${error?.message || 'Unknown error'}` })); - ws.close(); - } - }, - open(socket: any) { - // Send exec start request (using shared helper) - const httpRequest = buildExecStartHttpRequest(execId, target); - socket.write(httpRequest); - } + // Direct Docker connection (unix or tcp/hawser-standard) + const exec = await createExecForWs(containerId, [shell], user, target); + const execId = exec.Id; + + let headersStripped = false; + const state = { isChunked: false }; + + // Create Node.js TCP/Unix socket connection to Docker + let dockerStream: net.Socket; + if (target.type === 'unix') { + dockerStream = net.createConnection({ path: target.socket }); + } else if (target.type === 'tcp' && target.tls) { + const tlsOpts: tls.ConnectionOptions = { + host: target.host, + port: target.port, + servername: target.host, + rejectUnauthorized: target.tls.rejectUnauthorized ?? true }; - - let dockerStream: any; - if (target.type === 'unix') { - dockerStream = await Bun.connect({ unix: target.socket, socket: socketHandler }); - } else if (target.type === 'tcp') { - // Build connection options with TLS if configured - const connectOpts: any = { hostname: target.host, port: target.port, socket: socketHandler }; - if (target.tls) { - connectOpts.tls = { - sessionTimeout: 0, // Disable TLS session caching for mTLS - servername: target.host, // Required for SNI - rejectUnauthorized: target.tls.rejectUnauthorized ?? true - }; - if (target.tls.ca) connectOpts.tls.ca = [target.tls.ca]; - if (target.tls.cert) connectOpts.tls.cert = [target.tls.cert]; - if (target.tls.key) connectOpts.tls.key = target.tls.key; - } - dockerStream = await Bun.connect(connectOpts); - } - - dockerStreams.set(connId, { stream: dockerStream, execId, target, state, ws }); - } catch (error: any) { - console.error('[Terminal WS] Connection error:', error?.message || error); - ws.send(JSON.stringify({ type: 'error', message: error.message })); - ws.close(); - } - }, - async message(ws, message) { - const url = new URL((ws.data as any).url, `http://localhost:${WS_PORT}`); - const connId = (ws.data as any).connId as string | undefined; - - // Handle Hawser Edge messages - if (url.pathname === '/api/hawser/connect') { - try { - // Debug: Log raw message info - const msgType = typeof message; - const msgLen = typeof message === 'string' ? message.length : - message instanceof ArrayBuffer ? message.byteLength : - (message as Buffer).length || 0; - console.log(`[Hawser WS] Received message: type=${msgType}, length=${msgLen}`); - - // Convert message to string properly (handles both string and ArrayBuffer) - let messageStr: string; - if (typeof message === 'string') { - messageStr = message; - } else if (message instanceof ArrayBuffer) { - messageStr = new TextDecoder().decode(message); - } else if (Buffer.isBuffer(message)) { - messageStr = message.toString('utf-8'); - } else { - // Uint8Array or similar - messageStr = new TextDecoder().decode(new Uint8Array(message as ArrayBuffer)); - } - - console.log(`[Hawser WS] Decoded string length: ${messageStr.length}`); - if (messageStr.length > 0) { - console.log(`[Hawser WS] First 200 chars: ${messageStr.slice(0, 200)}`); - } - - const msg = JSON.parse(messageStr); - console.log(`[Hawser WS] Parsed message type: ${msg.type}`); - await handleHawserMessage(ws, msg); - } catch (error: any) { - console.error('[Hawser WS] Error handling message:', error.message); - // More detailed debug output - const msgType = typeof message; - const msgLen = typeof message === 'string' ? message.length : - message instanceof ArrayBuffer ? message.byteLength : - (message as Buffer).length || 0; - console.error(`[Hawser WS] Message details: type=${msgType}, length=${msgLen}`); - if (typeof message === 'string' && message.length > 0) { - console.error(`[Hawser WS] Message preview: ${message.slice(0, 500)}`); - } else if (message instanceof ArrayBuffer && message.byteLength > 0) { - const preview = new TextDecoder().decode(message.slice(0, 500)); - console.error(`[Hawser WS] ArrayBuffer preview: ${preview}`); - } else if (Buffer.isBuffer(message) && message.length > 0) { - console.error(`[Hawser WS] Buffer preview: ${message.toString('utf-8').slice(0, 500)}`); - } - ws.send(JSON.stringify({ type: 'error', error: error.message })); - } - return; + if (target.tls.ca) tlsOpts.ca = [target.tls.ca]; + if (target.tls.cert) tlsOpts.cert = [target.tls.cert]; + if (target.tls.key) tlsOpts.key = target.tls.key; + dockerStream = tls.connect(tlsOpts); + } else { + dockerStream = net.createConnection({ host: target.host, port: target.port }); } - // Check if this is an Edge exec session - const edgeExecId = (ws.data as any)?.edgeExecId; - if (edgeExecId) { - const session = edgeExecSessions.get(edgeExecId); - if (session) { - const conn = edgeConnections.get(session.environmentId); - if (conn) { - try { - const msg = JSON.parse(message.toString()); - if (msg.type === 'input') { - // Forward input to agent (using shared helper) - conn.ws.send(JSON.stringify(createExecInputMessage(edgeExecId, msg.data))); - } else if (msg.type === 'resize') { - // Forward resize to agent (using shared helper) - conn.ws.send(JSON.stringify(createExecResizeMessage(edgeExecId, msg.cols, msg.rows))); - } - } catch (e) { - console.error('[Terminal WS] Error handling Edge message:', e); + dockerStream.on('connect', () => { + const httpRequest = buildExecStartHttpRequest(execId, target); + dockerStream.write(httpRequest); + }); + + dockerStream.on('data', (data: Buffer) => { + if (ws.readyState === WsWebSocket.OPEN) { + let text = data.toString('utf-8'); + if (!headersStripped) { + if (text.toLowerCase().includes('transfer-encoding: chunked')) { + state.isChunked = true; + } + const headerEnd = text.indexOf('\r\n\r\n'); + if (headerEnd > -1) { + text = text.slice(headerEnd + 4); + headersStripped = true; + } else if (text.startsWith('HTTP/')) { + return; } } + if (state.isChunked && text) { + text = text.replace(/^[0-9a-fA-F]+\r\n/gm, '').replace(/\r\n$/g, ''); + } + if (text) { + ws.send(JSON.stringify({ type: 'output', data: text })); + } } - return; - } + }); - // Terminal message handling (direct Docker connection) - if (!connId) return; - const d = dockerStreams.get(connId); - if (!d) return; + dockerStream.on('close', () => { + if (ws.readyState === WsWebSocket.OPEN) { + ws.send(JSON.stringify({ type: 'exit' })); + ws.close(); + } + }); + dockerStream.on('error', (error: Error) => { + console.error('[Terminal WS] Socket error:', error?.message || error); + if (ws.readyState === WsWebSocket.OPEN) { + ws.send(JSON.stringify({ type: 'error', message: `Connection error: ${error?.message || 'Unknown error'}` })); + } + }); + + dockerStreams.set(connId, { stream: dockerStream, execId, target, state, ws }); + } catch (error: any) { + console.error('[Terminal WS] Connection error:', error?.message || error); + ws.send(JSON.stringify({ type: 'error', message: error.message })); + ws.close(); + } + })(); + + // Handle messages + ws.on('message', async (message: Buffer | string) => { + const meta = wsMetadata.get(ws); + if (!meta) return; + const wsUrl = new URL(meta.url, `http://localhost:${WS_PORT}`); + + // Handle Hawser Edge messages + if (wsUrl.pathname === '/api/hawser/connect') { try { - const msg = JSON.parse(message.toString()); - if (msg.type === 'input' && d.stream) { - // Always write raw input - chunked encoding only affects reading output - d.stream.write(msg.data); - } else if (msg.type === 'resize' && d.execId) { - resizeExecForWs(d.execId, msg.cols, msg.rows, d.target); - } - } catch { - // If not JSON, treat as raw input - if (d.stream) { - d.stream.write(message); - } + const messageStr = typeof message === 'string' ? message : message.toString('utf-8'); + const msg = JSON.parse(messageStr); + await handleHawserMessage(ws, msg); + } catch (error: any) { + console.error('[Hawser WS] Error handling message:', error.message); + ws.send(JSON.stringify({ type: 'error', error: error.message })); } - }, - close(ws) { - // Check if it's a Hawser connection - const envId = wsToEnvId.get(ws); - if (envId) { - const conn = edgeConnections.get(envId); + return; + } + + // Check if this is an Edge exec session + const edgeExecId = meta.edgeExecId; + if (edgeExecId) { + const session = edgeExecSessions.get(edgeExecId); + if (session) { + const conn = edgeConnections.get(session.environmentId); if (conn) { - console.log(`[Hawser WS] Agent disconnected: ${conn.agentId}`); - // Clear server-side ping interval - if (conn.pingInterval) { - clearInterval(conn.pingInterval); - conn.pingInterval = undefined; + try { + const msg = JSON.parse(message.toString()); + if (msg.type === 'input') { + conn.ws.send(JSON.stringify(createExecInputMessage(edgeExecId, msg.data))); + } else if (msg.type === 'resize') { + conn.ws.send(JSON.stringify(createExecResizeMessage(edgeExecId, msg.cols, msg.rows))); + } + } catch (e) { + console.error('[Terminal WS] Error handling Edge message:', e); } - // Reject pending requests - for (const [, pending] of conn.pendingRequests) { - clearTimeout(pending.timeout); - pending.reject(new Error('Connection closed')); - } - // Clean up pending stream requests - for (const [, pending] of conn.pendingStreamRequests) { - pending.onEnd('Connection closed'); - } - edgeConnections.delete(envId); } - wsToEnvId.delete(ws); - return; } + return; + } - // Check if it's an Edge exec session - const edgeExecId = (ws.data as any)?.edgeExecId; - if (edgeExecId) { - const session = edgeExecSessions.get(edgeExecId); - if (session) { - // Send exec_end to agent (using shared helper) - const conn = edgeConnections.get(session.environmentId); - if (conn) { - conn.ws.send(JSON.stringify(createExecEndMessage(edgeExecId))); - } - edgeExecSessions.delete(edgeExecId); - console.log(`[Terminal WS] Edge exec session closed: ${edgeExecId}`); - } - return; + // Terminal message handling (direct Docker connection) + const connId = meta.connId; + if (!connId) return; + const d = dockerStreams.get(connId); + if (!d) return; + + try { + const msg = JSON.parse(message.toString()); + if (msg.type === 'input' && d.stream) { + d.stream.write(msg.data); + } else if (msg.type === 'resize' && d.execId) { + resizeExecForWs(d.execId, msg.cols, msg.rows, d.target); } - - // Terminal connection cleanup (direct Docker) - const connId = (ws.data as any)?.connId as string | undefined; - if (connId) { - const d = dockerStreams.get(connId); - if (d?.stream) { - d.stream.end(); - } - dockerStreams.delete(connId); + } catch { + if (d.stream) { + d.stream.write(message); } } - } + }); + + // Handle close + ws.on('close', () => { + const meta = wsMetadata.get(ws); + wsMetadata.delete(ws); + + // Check if it's a Hawser connection + const envId = wsToEnvId.get(ws); + if (envId) { + const conn = edgeConnections.get(envId); + if (conn) { + console.log(`[Hawser WS] Agent disconnected: ${conn.agentId}`); + if (conn.pingInterval) { + clearInterval(conn.pingInterval); + conn.pingInterval = undefined; + } + for (const [, pending] of conn.pendingRequests) { + clearTimeout(pending.timeout); + pending.reject(new Error('Connection closed')); + } + for (const [, pending] of conn.pendingStreamRequests) { + pending.onEnd('Connection closed'); + } + edgeConnections.delete(envId); + } + wsToEnvId.delete(ws); + return; + } + + // Check if it's an Edge exec session + const edgeExecId = meta?.edgeExecId; + if (edgeExecId) { + const session = edgeExecSessions.get(edgeExecId); + if (session) { + const conn = edgeConnections.get(session.environmentId); + if (conn) { + conn.ws.send(JSON.stringify(createExecEndMessage(edgeExecId))); + } + edgeExecSessions.delete(edgeExecId); + console.log(`[Terminal WS] Edge exec session closed: ${edgeExecId}`); + } + return; + } + + // Terminal connection cleanup (direct Docker) + const connId = meta?.connId; + if (connId) { + const d = dockerStreams.get(connId); + if (d?.stream) { + d.stream.end(); + } + dockerStreams.delete(connId); + } + }); }); - console.log(`[Terminal WS] WebSocket server running on port ${WS_PORT}`); + httpServer.listen(WS_PORT, () => { + console.log(`[Terminal WS] WebSocket server running on port ${WS_PORT}`); + }); } }; } +// Rate limiter for failed Hawser token auth (dev mode) +const hawserAuthFailCache = new Map(); +const HAWSER_AUTH_FAIL_COOLDOWN_MS = 60_000; + +// ─── Reconnection storm throttle (mirrors hawser.ts) ─── +interface ReconnectTrackerEntry { + timestamps: number[]; + cooldownUntil: number; + cooldownLevel: number; +} +const reconnectTracker = new Map(); +const RECONNECT_WINDOW_MS = 2 * 60 * 1000; +const RECONNECT_BURST = 3; +const COOLDOWN_LEVELS_SECS = [30, 60, 120, 300]; +const STABLE_THRESHOLD_MS = 5 * 60 * 1000; +const STALE_TRACKER_MS = 10 * 60 * 1000; + +function recordReconnection(envId: number): { allowed: true } | { allowed: false; retryAfter: number } { + const now = Date.now(); + let entry = reconnectTracker.get(envId); + + if (!entry) { + entry = { timestamps: [now], cooldownUntil: 0, cooldownLevel: 0 }; + reconnectTracker.set(envId, entry); + return { allowed: true }; + } + + if (now < entry.cooldownUntil) { + const retryAfter = Math.ceil((entry.cooldownUntil - now) / 1000); + return { allowed: false, retryAfter }; + } + + entry.timestamps = entry.timestamps.filter(ts => now - ts < RECONNECT_WINDOW_MS); + entry.timestamps.push(now); + + if (entry.timestamps.length > RECONNECT_BURST) { + const level = Math.min(entry.cooldownLevel, COOLDOWN_LEVELS_SECS.length - 1); + const cooldownSecs = COOLDOWN_LEVELS_SECS[level]; + entry.cooldownUntil = now + cooldownSecs * 1000; + entry.cooldownLevel = Math.min(entry.cooldownLevel + 1, COOLDOWN_LEVELS_SECS.length - 1); + + console.warn( + `[Hawser WS] Reconnection storm detected for env ${envId}: ` + + `${entry.timestamps.length} connections in ${RECONNECT_WINDOW_MS / 1000}s. ` + + `Cooldown ${cooldownSecs}s (level ${level})` + ); + + return { allowed: false, retryAfter: cooldownSecs }; + } + + return { allowed: true }; +} + // Handle Hawser Edge protocol messages async function handleHawserMessage(ws: any, msg: any) { if (msg.type === 'hello') { - // Validate token using the app's hawser module - // For dev mode, we'll do a simplified validation - console.log(`[Hawser WS] Hello from agent: ${msg.agentName} (${msg.agentId})`); + const agentId = msg.agentId || 'unknown'; + console.log(`[Hawser WS] Hello from agent: ${msg.agentName} (${agentId})`); + + // Rate-limit agents that recently failed auth - skip expensive Argon2id verification + const lastFail = hawserAuthFailCache.get(agentId); + if (lastFail && (Date.now() - lastFail) < HAWSER_AUTH_FAIL_COOLDOWN_MS) { + ws.send(JSON.stringify({ type: 'error', error: 'Rate limited - retry later' })); + ws.close(); + return; + } // In dev mode, we need to validate the token against the database const db = getDb(); @@ -885,7 +894,7 @@ async function handleHawserMessage(ws: any, msg: any) { let matchedToken: any = null; for (const t of tokens) { try { - const isValid = await Bun.password.verify(msg.token, t.token); + const isValid = await argon2.verify(t.token, msg.token); if (isValid) { matchedToken = t; break; @@ -897,13 +906,29 @@ async function handleHawserMessage(ws: any, msg: any) { if (!matchedToken) { console.log('[Hawser WS] Invalid token'); + hawserAuthFailCache.set(agentId, Date.now()); ws.send(JSON.stringify({ type: 'error', error: 'Invalid token' })); ws.close(); return; } + // Clear any previous failure on successful auth + hawserAuthFailCache.delete(agentId); const environmentId = matchedToken.environment_id; + // Throttle reconnection storms + const throttle = recordReconnection(environmentId); + if (!throttle.allowed) { + console.log(`[Hawser WS] Throttling reconnection for env ${environmentId}: retry after ${throttle.retryAfter}s`); + ws.send(JSON.stringify({ + type: 'error', + error: `Reconnection throttled. Retry after ${throttle.retryAfter}s.`, + retryAfter: throttle.retryAfter + })); + ws.close(); + return; + } + // Update environment with agent info try { db.prepare(`UPDATE environments SET @@ -946,7 +971,16 @@ async function handleHawserMessage(ws: any, msg: any) { existing.pendingRequests.clear(); existing.pendingStreamRequests.clear(); - existing.ws.close(1000, 'Replaced by new connection'); + if (existing.pingInterval) { + clearInterval(existing.pingInterval); + existing.pingInterval = undefined; + } + // Immediately destroy TCP socket — no graceful close needed for replaced connections + if (typeof existing.ws.terminate === 'function') { + existing.ws.terminate(); + } else { + existing.ws.close(1000, 'Replaced by new connection'); + } wsToEnvId.delete(existing.ws); } @@ -961,7 +995,7 @@ async function handleHawserMessage(ws: any, msg: any) { hostname: msg.hostname || 'unknown', capabilities: msg.capabilities || [], connectedAt: new Date(), - lastHeartbeat: new Date(), + lastHeartbeat: Date.now(), pendingRequests: new Map(), pendingStreamRequests: new Map() }; @@ -997,7 +1031,7 @@ async function handleHawserMessage(ws: any, msg: any) { if (envId) { const conn = edgeConnections.get(envId); if (conn) { - conn.lastHeartbeat = new Date(); + conn.lastHeartbeat = Date.now(); } } ws.send(JSON.stringify({ type: 'pong', timestamp: Date.now() })); @@ -1007,7 +1041,7 @@ async function handleHawserMessage(ws: any, msg: any) { if (envId) { const conn = edgeConnections.get(envId); if (conn) { - conn.lastHeartbeat = new Date(); + conn.lastHeartbeat = Date.now(); } } } else if (msg.type === 'response') { @@ -1129,7 +1163,7 @@ async function handleHawserMessage(ws: any, msg: any) { } export default defineConfig({ - plugins: [bunExternals(), tailwindcss(), sveltekit(), webSocketPlugin()], + plugins: [tailwindcss(), sveltekit(), webSocketPlugin()], define: { __BUILD_DATE__: JSON.stringify(new Date().toISOString()), __BUILD_COMMIT__: JSON.stringify(getGitCommit()), @@ -1151,12 +1185,6 @@ export default defineConfig({ build: { target: 'esnext', minify: 'esbuild', - sourcemap: false, - rollupOptions: { - external: [/^bun:/] - } - }, - ssr: { - external: [/^bun:/] + sourcemap: false } });