Skip to content

ingress: allow per-port listen-address override (tailnet-only ingresses) #271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
rgarcia wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

ingress: allow per-port listen-address override (tailnet-only ingresses) #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions cmd/api/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,20 @@ type CaddyConfig struct {
AdminPort int `koanf:"admin_port"`
InternalDNSPort int `koanf:"internal_dns_port"`
StopOnShutdown bool `koanf:"stop_on_shutdown"`

// PortListenAddresses overrides the listen address for specific listen ports.
// Keyed by listen port, value is the bind address (e.g. a Tailscale IP).
// Ports not present in the map fall back to ListenAddress (default "0.0.0.0").
//
// Use case: bind CDP/ChromeDriver ingresses to a non-public (e.g. Tailscale)
// interface so they are not reachable on the public NIC, while keeping
// 443/444 public. Example (YAML):
//
// caddy:
// port_listen_addresses:
// 9222: "100.107.186.40"
// 9224: "100.107.186.40"
PortListenAddresses map[int]string `koanf:"port_listen_addresses"`
}

// ACMEConfig holds ACME / TLS certificate settings.
Expand Down
38 changes: 38 additions & 0 deletions cmd/api/config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,3 +455,41 @@ func TestValidateAllowsDisabledSnapshotCompressionDefaultWithoutValidAlgorithm(t
t.Fatalf("expected disabled snapshot compression default to ignore algorithm/level, got %v", err)
}
}

func TestDefaultConfigHasNoCaddyPortListenAddresses(t *testing.T) {
cfg := defaultConfig()
if cfg.Caddy.ListenAddress != "0.0.0.0" {
t.Fatalf("expected default caddy.listen_address to remain 0.0.0.0, got %q", cfg.Caddy.ListenAddress)
}
if len(cfg.Caddy.PortListenAddresses) != 0 {
t.Fatalf("expected default caddy.port_listen_addresses to be empty, got %v", cfg.Caddy.PortListenAddresses)
}
}

func TestLoadCaddyPortListenAddressesFromYAML(t *testing.T) {
tmp := t.TempDir()
cfgPath := filepath.Join(tmp, "config.yaml")
yaml := "caddy:\n port_listen_addresses:\n 9222: \"100.107.186.40\"\n 9224: \"100.107.186.40\"\n"
if err := os.WriteFile(cfgPath, []byte(yaml), 0600); err != nil {
t.Fatalf("write temp config: %v", err)
}

cfg, err := Load(cfgPath)
if err != nil {
t.Fatalf("load config: %v", err)
}

// Global listen address must remain the default (public).
if cfg.Caddy.ListenAddress != "0.0.0.0" {
t.Fatalf("expected caddy.listen_address to remain 0.0.0.0, got %q", cfg.Caddy.ListenAddress)
}
if got := cfg.Caddy.PortListenAddresses[9222]; got != "100.107.186.40" {
t.Fatalf("expected caddy.port_listen_addresses[9222] to be 100.107.186.40, got %q", got)
}
if got := cfg.Caddy.PortListenAddresses[9224]; got != "100.107.186.40" {
t.Fatalf("expected caddy.port_listen_addresses[9224] to be 100.107.186.40, got %q", got)
}
if len(cfg.Caddy.PortListenAddresses) != 2 {
t.Fatalf("expected exactly 2 port overrides, got %v", cfg.Caddy.PortListenAddresses)
}
}
7 changes: 7 additions & 0 deletions config.example.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,13 @@ data_dir: /var/lib/hypeman
# admin_port: 0 # 0 = random (for dev); install script sets to 2019 for production
# internal_dns_port: 0 # 0 = random (for dev); install script sets to 5353 for production
# stop_on_shutdown: false # Set to true if you want Caddy to stop when hypeman stops
# # Override the bind address for specific listen ports. Ports not listed here
# # fall back to listen_address (above). Useful to make selected ingresses
# # (e.g. CDP/ChromeDriver) reachable only on a non-public interface such as a
# # Tailscale IP, while keeping 443/444 public. Unset = every port uses listen_address.
# port_listen_addresses:
# 9222: "100.107.186.40" # CDP -> tailnet only
# 9224: "100.107.186.40" # ChromeDriver -> tailnet only

# =============================================================================
# TLS / ACME Configuration (for HTTPS ingresses)
Expand Down
36 changes: 27 additions & 9 deletions lib/ingress/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,19 +172,37 @@ type CaddyConfigGenerator struct {
acme ACMEConfig
apiIngress APIIngressConfig
dnsResolverPort int

// portListenAddresses overrides the bind address for specific listen ports.
// Keyed by listen port; ports not present fall back to listenAddress.
// Used to bind selected ingresses (e.g. CDP/ChromeDriver) to a non-public
// interface such as a Tailscale IP. Empty/nil means every port uses
// listenAddress (default behavior).
portListenAddresses map[int]string
}

// NewCaddyConfigGenerator creates a new Caddy config generator.
func NewCaddyConfigGenerator(p *paths.Paths, listenAddress string, adminAddress string, adminPort int, acme ACMEConfig, apiIngress APIIngressConfig, dnsResolverPort int) *CaddyConfigGenerator {
func NewCaddyConfigGenerator(p *paths.Paths, listenAddress string, adminAddress string, adminPort int, acme ACMEConfig, apiIngress APIIngressConfig, dnsResolverPort int, portListenAddresses map[int]string) *CaddyConfigGenerator {
return &CaddyConfigGenerator{
paths: p,
listenAddress: listenAddress,
adminAddress: adminAddress,
adminPort: adminPort,
acme: acme,
apiIngress: apiIngress,
dnsResolverPort: dnsResolverPort,
paths: p,
listenAddress: listenAddress,
adminAddress: adminAddress,
adminPort: adminPort,
acme: acme,
apiIngress: apiIngress,
dnsResolverPort: dnsResolverPort,
portListenAddresses: portListenAddresses,
}
}

// listenAddressForPort returns the bind address Caddy should use for the given
// listen port. If a per-port override is configured for the port, it is used;
// otherwise the global listen address is returned.
func (g *CaddyConfigGenerator) listenAddressForPort(port int) string {
if addr, ok := g.portListenAddresses[port]; ok && addr != "" {
return addr
}
return g.listenAddress
}

// GenerateConfig generates the Caddy JSON configuration.
Expand Down Expand Up @@ -400,7 +418,7 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr
allRoutes := append(routes, catchAllRoute)

server := map[string]interface{}{
"listen": []string{fmt.Sprintf("%s:%d", g.listenAddress, port)},
"listen": []string{fmt.Sprintf("%s:%d", g.listenAddressForPort(port), port)},
"routes": allRoutes,
"logs": map[string]interface{}{}, // Disable access logs
}
Expand Down
138 changes: 131 additions & 7 deletions lib/ingress/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package ingress
import (
"context"
"encoding/json"
"fmt"
"os"
"testing"

Expand All @@ -27,7 +28,7 @@ func setupTestGenerator(t *testing.T) (*CaddyConfigGenerator, *paths.Paths, func
// Empty ACMEConfig means TLS is not configured
// Use DNS resolver port for dynamic upstreams
dnsResolverPort := 5353
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, ACMEConfig{}, APIIngressConfig{}, dnsResolverPort)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, ACMEConfig{}, APIIngressConfig{}, dnsResolverPort, nil)

cleanup := func() {
os.RemoveAll(tmpDir)
Expand Down Expand Up @@ -81,7 +82,7 @@ func TestGenerateConfig_StoragePath(t *testing.T) {
require.NoError(t, os.MkdirAll(p.CaddyDir(), 0755))
require.NoError(t, os.MkdirAll(p.CaddyDataDir(), 0755))

generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, ACMEConfig{}, APIIngressConfig{}, 5353)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, ACMEConfig{}, APIIngressConfig{}, 5353, nil)

ctx := context.Background()
data, err := generator.GenerateConfig(ctx, []Ingress{})
Expand Down Expand Up @@ -417,7 +418,7 @@ func TestGenerateConfig_WithTLS(t *testing.T) {
DNSProvider: DNSProviderCloudflare,
CloudflareAPIToken: "test-token",
}
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, acmeConfig, APIIngressConfig{}, 5353)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, acmeConfig, APIIngressConfig{}, 5353, nil)

ctx := context.Background()
ingresses := []Ingress{
Expand Down Expand Up @@ -702,7 +703,7 @@ func TestGenerateConfig_MixedTLSAndNonTLS(t *testing.T) {
DNSProvider: DNSProviderCloudflare,
CloudflareAPIToken: "test-token",
}
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, acmeConfig, APIIngressConfig{}, 5353)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, acmeConfig, APIIngressConfig{}, 5353, nil)

ctx := context.Background()
ingresses := []Ingress{
Expand Down Expand Up @@ -841,7 +842,7 @@ func TestGenerateConfig_TLSHostnameDeduplication(t *testing.T) {
CloudflareAPIToken: "test-token",
AllowedDomains: "*.example.com",
}
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, acmeConfig, APIIngressConfig{}, 5353)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, acmeConfig, APIIngressConfig{}, 5353, nil)

ctx := context.Background()
// Create two ingresses with the same wildcard hostname pattern on different ports
Expand Down Expand Up @@ -930,7 +931,7 @@ func TestGenerateConfig_PortIsolation(t *testing.T) {
CloudflareAPIToken: "test-token",
AllowedDomains: "*.example.com",
}
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, acmeConfig, APIIngressConfig{}, 5353)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, acmeConfig, APIIngressConfig{}, 5353, nil)

ctx := context.Background()
// Create wildcard ingresses on different ports that would conflict
Expand Down Expand Up @@ -1061,7 +1062,7 @@ func TestGenerateConfig_DynamicUpstreams(t *testing.T) {
require.NoError(t, os.MkdirAll(p.CaddyDataDir(), 0755))

dnsPort := 5353
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, ACMEConfig{}, APIIngressConfig{}, dnsPort)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, ACMEConfig{}, APIIngressConfig{}, dnsPort, nil)

ctx := context.Background()
ingresses := []Ingress{
Expand Down Expand Up @@ -1092,3 +1093,126 @@ func TestGenerateConfig_DynamicUpstreams(t *testing.T) {
assert.Contains(t, configStr, "resolver")
assert.Contains(t, configStr, "127.0.0.1:5353")
}

// TestGenerateConfig_PortListenAddressOverride verifies that per-port listen
// address overrides bind designated ports (e.g. CDP/ChromeDriver) to a
// non-public interface while other ports keep the global listen address.
func TestGenerateConfig_PortListenAddressOverride(t *testing.T) {
// Ingresses spanning four ports: 444 (browser API), 443 (VNC),
// 9222 (CDP), 9224 (ChromeDriver). 9222/9224 should be tailnet-only.
ingresses := []Ingress{
{
ID: "ing-api",
Name: "browser-api",
Rules: []IngressRule{
{Match: IngressMatch{Hostname: "inst.host.kernel.sh", Port: 444}, Target: IngressTarget{Instance: "inst", Port: 10001}},
},
},
{
ID: "ing-vnc",
Name: "vnc",
Rules: []IngressRule{
{Match: IngressMatch{Hostname: "inst.host.kernel.sh", Port: 443}, Target: IngressTarget{Instance: "inst", Port: 6901}},
},
},
{
ID: "ing-cdp",
Name: "cdp",
Rules: []IngressRule{
{Match: IngressMatch{Hostname: "inst.host.kernel.sh", Port: 9222}, Target: IngressTarget{Instance: "inst", Port: 9222}},
},
},
{
ID: "ing-chromedriver",
Name: "chromedriver",
Rules: []IngressRule{
{Match: IngressMatch{Hostname: "inst.host.kernel.sh", Port: 9224}, Target: IngressTarget{Instance: "inst", Port: 9224}},
},
},
}

const tsIP = "100.107.186.40"

tests := []struct {
name string
globalListenAddress string
portListenAddresses map[int]string
// wantListen maps listen port -> expected "addr:port" for that server.
wantListen map[int]string
}{
{
name: "no override uses global address for all ports (backward compatible)",
globalListenAddress: "0.0.0.0",
portListenAddresses: nil,
wantListen: map[int]string{
444: "0.0.0.0:444",
443: "0.0.0.0:443",
9222: "0.0.0.0:9222",
9224: "0.0.0.0:9224",
},
},
{
name: "empty map uses global address for all ports",
globalListenAddress: "0.0.0.0",
portListenAddresses: map[int]string{},
wantListen: map[int]string{
444: "0.0.0.0:444",
443: "0.0.0.0:443",
9222: "0.0.0.0:9222",
9224: "0.0.0.0:9224",
},
},
{
name: "CDP and ChromeDriver bound to tailscale IP, 443/444 stay public",
globalListenAddress: "0.0.0.0",
portListenAddresses: map[int]string{9222: tsIP, 9224: tsIP},
wantListen: map[int]string{
444: "0.0.0.0:444",
443: "0.0.0.0:443",
9222: tsIP + ":9222",
9224: tsIP + ":9224",
},
},
{
name: "empty override value falls back to global address",
globalListenAddress: "0.0.0.0",
portListenAddresses: map[int]string{9222: ""},
wantListen: map[int]string{
9222: "0.0.0.0:9222",
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "ingress-config-portbind-test-*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)

p := paths.New(tmpDir)
require.NoError(t, os.MkdirAll(p.CaddyDir(), 0755))
require.NoError(t, os.MkdirAll(p.CaddyDataDir(), 0755))

generator := NewCaddyConfigGenerator(p, tt.globalListenAddress, "127.0.0.1", 2019, ACMEConfig{}, APIIngressConfig{}, 5353, tt.portListenAddresses)

data, err := generator.GenerateConfig(context.Background(), ingresses)
require.NoError(t, err)

var config map[string]interface{}
require.NoError(t, json.Unmarshal(data, &config))

apps := config["apps"].(map[string]interface{})
httpApp := apps["http"].(map[string]interface{})
servers := httpApp["servers"].(map[string]interface{})

for port, wantAddr := range tt.wantListen {
serverName := fmt.Sprintf("ingress-%d", port)
server, ok := servers[serverName].(map[string]interface{})
require.True(t, ok, "expected server %q to exist", serverName)
listen := server["listen"].([]interface{})
require.Len(t, listen, 1)
assert.Equal(t, wantAddr, listen[0].(string), "listen address for port %d", port)
}
})
}
}
9 changes: 9 additions & 0 deletions lib/ingress/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,13 @@ type Config struct {
// When false, Caddy continues running independently.
StopOnShutdown bool

// PortListenAddresses overrides the bind address for specific listen ports.
// Keyed by listen port; ports not present fall back to ListenAddress.
// Used to bind selected ingresses (e.g. CDP/ChromeDriver) to a non-public
// interface such as a Tailscale IP. Nil/empty means every port uses
// ListenAddress (default behavior).
PortListenAddresses map[int]string

// ACME configuration for TLS certificates
ACME ACMEConfig

Expand Down Expand Up @@ -140,6 +147,7 @@ func NewManager(p *paths.Paths, config Config, instanceResolver InstanceResolver
config.ACME,
config.APIIngress,
dnsServer.Port(),
config.PortListenAddresses,
)

return &manager{
Expand Down Expand Up @@ -193,6 +201,7 @@ func (m *manager) Initialize(ctx context.Context) error {
m.config.ACME,
m.config.APIIngress,
m.dnsServer.Port(),
m.config.PortListenAddresses,
)

// Load existing ingresses
Expand Down
6 changes: 3 additions & 3 deletions lib/ingress/validation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ func TestConfigGeneration(t *testing.T) {

// Create config generator with DNS-based dynamic upstream settings
dnsResolverPort := 5353
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", adminPort, ACMEConfig{}, APIIngressConfig{}, dnsResolverPort)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", adminPort, ACMEConfig{}, APIIngressConfig{}, dnsResolverPort, nil)

ctx := context.Background()

Expand Down Expand Up @@ -367,7 +367,7 @@ func TestTLSConfigGeneration(t *testing.T) {
DNSProvider: DNSProviderCloudflare,
CloudflareAPIToken: "test-token",
}
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", adminPort, acmeConfig, APIIngressConfig{}, dnsResolverPort)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", adminPort, acmeConfig, APIIngressConfig{}, dnsResolverPort, nil)

ingresses := []Ingress{
{
Expand Down Expand Up @@ -404,7 +404,7 @@ func TestTLSConfigGeneration(t *testing.T) {

t.Run("NoTLSAutomationWithoutConfig", func(t *testing.T) {
// Empty ACME config
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", adminPort, ACMEConfig{}, APIIngressConfig{}, dnsResolverPort)
generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", adminPort, ACMEConfig{}, APIIngressConfig{}, dnsResolverPort, nil)

ingresses := []Ingress{
{
Expand Down
Loading
Loading