feat(bootstrap): support multiple upstreams

If more than one upstream is configured, they are raced via
a `ParallelBestResolver`.
This commit is contained in:
ThinkChaos 2022-12-02 22:21:12 -05:00
parent fb009053bf
commit a79459987b
9 changed files with 299 additions and 111 deletions

View File

@ -10,11 +10,13 @@ import (
var _ = Describe("Serve command", func() {
When("Serve command is called", func() {
It("should start DNS server", func() {
config.GetConfig().BootstrapDNS = config.BootstrapConfig{
Upstream: config.Upstream{
Net: config.NetProtocolTcpTls,
Host: "1.1.1.1",
Port: 53,
config.GetConfig().BootstrapDNS = []config.BootstrappedUpstreamConfig{
{
Upstream: config.Upstream{
Net: config.NetProtocolTcpTls,
Host: "1.1.1.1",
Port: 53,
},
},
}

View File

@ -124,8 +124,8 @@ func (s *QTypeSet) Insert(qType dns.Type) {
type Duration time.Duration
func (c *Duration) String() string {
return durafmt.Parse(time.Duration(*c)).String()
func (c Duration) String() string {
return durafmt.Parse(time.Duration(c)).String()
}
//nolint:gochecknoglobals
@ -150,7 +150,7 @@ func (u *Upstream) IsDefault() bool {
}
// String returns the string representation of u
func (u *Upstream) String() string {
func (u Upstream) String() string {
if u.IsDefault() {
return "no upstream"
}
@ -217,20 +217,41 @@ func (l *ListenConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
return nil
}
// UnmarshalYAML creates BootstrapDNSConfig from YAML
func (b *BootstrapDNSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
var single BootstrappedUpstreamConfig
if err := unmarshal(&single); err == nil {
*b = BootstrapDNSConfig{single}
return nil
}
// bootstrapDNSConfig is used to avoid infinite recursion:
// if we used BootstrapDNSConfig, unmarshal would just call us again.
var c bootstrapDNSConfig
if err := unmarshal(&c); err != nil {
return err
}
*b = BootstrapDNSConfig(c)
return nil
}
// UnmarshalYAML creates BootstrapConfig from YAML
func (b *BootstrapConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
func (b *BootstrappedUpstreamConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
if err := unmarshal(&b.Upstream); err == nil {
return nil
}
// bootstrapConfig is used to avoid infinite recursion:
// if we used BootstrapConfig, unmarshal would just call us again.
var c bootstrapConfig
var c bootstrappedUpstreamConfig
if err := unmarshal(&c); err != nil {
return err
}
*b = BootstrapConfig(c)
*b = BootstrappedUpstreamConfig(c)
return nil
}
@ -471,7 +492,7 @@ type Config struct {
StartVerifyUpstream bool `yaml:"startVerifyUpstream" default:"false"`
CertFile string `yaml:"certFile"`
KeyFile string `yaml:"keyFile"`
BootstrapDNS BootstrapConfig `yaml:"bootstrapDns"`
BootstrapDNS BootstrapDNSConfig `yaml:"bootstrapDns"`
HostsFile HostsFileConfig `yaml:"hostsFile"`
FqdnOnly bool `yaml:"fqdnOnly" default:"false"`
Filtering FilteringConfig `yaml:"filtering"`
@ -503,9 +524,16 @@ type PortsConfig struct {
TLS ListenConfig `yaml:"tls"`
}
// split in two types to avoid infinite recursion. See `BootstrapDNSConfig.UnmarshalYAML`.
type (
BootstrapConfig bootstrapConfig // to avoid infinite recursion. See BootstrapConfig.UnmarshalYAML.
bootstrapConfig struct {
BootstrapDNSConfig bootstrapDNSConfig
bootstrapDNSConfig []BootstrappedUpstreamConfig
)
// split in two types to avoid infinite recursion. See `BootstrappedUpstreamConfig.UnmarshalYAML`.
type (
BootstrappedUpstreamConfig bootstrappedUpstreamConfig
bootstrappedUpstreamConfig struct {
Upstream Upstream `yaml:"upstream"`
IPs []net.IP `yaml:"ips"`
}

View File

@ -260,7 +260,7 @@ var _ = Describe("Config", func() {
err := unmarshalConfig([]byte(data), &cfg)
Expect(err).ShouldNot(HaveOccurred())
Expect(cfg.BootstrapDNS.Upstream.Host).Should(Equal("0.0.0.0"))
Expect(cfg.BootstrapDNS[0].Upstream.Host).Should(Equal("0.0.0.0"))
})
It("should be backwards compatible", func() {
cfg := Config{}
@ -272,8 +272,8 @@ bootstrapDns:
`
err := unmarshalConfig([]byte(data), &cfg)
Expect(err).ShouldNot(HaveOccurred())
Expect(cfg.BootstrapDNS.Upstream.Host).Should(Equal("dns.example.com"))
Expect(cfg.BootstrapDNS.IPs).Should(HaveLen(1))
Expect(cfg.BootstrapDNS[0].Upstream.Host).Should(Equal("dns.example.com"))
Expect(cfg.BootstrapDNS[0].IPs).Should(HaveLen(1))
})
})

View File

@ -213,11 +213,18 @@ minTlsServeVersion: 1.3
# if https port > 0: path to cert and key file for SSL encryption. if not set, self-signed certificate will be generated
#certFile: server.crt
#keyFile: server.key
# optional: use this DNS server to resolve blacklist urls and upstream DNS servers. Useful if no DNS resolver is configured and blocky needs to resolve a host name. Format net:IP:port, net must be udp or tcp
bootstrapDns: tcp+udp:1.1.1.1
# optional: use these DNS servers to resolve blacklist urls and upstream DNS servers. It is useful if no system DNS resolver is configured, and/or to encrypt the bootstrap queries.
bootstrapDns:
- tcp+udp:1.1.1.1
- usptream: https://1.1.1.1/dns-query
ips:
- 1.1.1.1
- upstream: https://dns.digitale-gesellschaft.ch/dns-query
ips:
- 185.95.218.42
filtering:
# optional: drop all queries with following query types. Default: empty
filtering:
queryTypes:
- AAAA
@ -241,7 +248,7 @@ ports:
# optional: Port(s) and optional bind ip address(es) to serve HTTPS used for prometheus metrics, pprof, REST API, DoH... If you wish to specify a specific IP, you can do so such as 192.168.0.1:443. Example: 443, :443, 127.0.0.1:443,[::1]:443
https: 443
# optional: Port(s) and optional bind ip address(es) to serve HTTP used for prometheus metrics, pprof, REST API, DoH... If you wish to specify a specific IP, you can do so such as 192.168.0.1:4000. Example: 4000, :4000, 127.0.0.1:4000,[::1]:4000
http: 4000
http: 4000
# optional: logging configuration
log:
@ -255,6 +262,6 @@ log:
privacy: false
# optional: add EDE error codes to dns response
ede:
ede:
# enabled if true, Default: false
enable: true

View File

@ -41,7 +41,7 @@ All logging port are optional.
!!! example
```yaml
ports:
ports:
dns: 53
http: 4000
https: 443
@ -147,12 +147,12 @@ value by setting the `upstreamTimeout` configuration parameter (in **duration fo
## Bootstrap DNS configuration
This DNS server is used to resolve upstream DoH and DoT servers that are specified as host names.
Useful if no system DNS resolver is configured, and to encrypt the bootstrap queries.
These DNS servers are used to resolve upstream DoH and DoT servers that are specified as host names, and list domains.
It is useful if no system DNS resolver is configured, and/or to encrypt the bootstrap queries.
| Parameter | Type | Mandatory | Default value | Description |
|-----------|----------------------|-----------------------------|---------------|--------------------------------------|
| upstream | Upstream (see below) | no | | |
| upstream | Upstream (see above) | no | | |
| ips | List of IPs | yes, if upstream is DoT/DoH | | Only valid if upstream is DoH or DoT |
If you only need to specify upstream, you can use the short form: `bootstrapDns: <upstream>`.
@ -165,9 +165,12 @@ If you only need to specify upstream, you can use the short form: `bootstrapDns:
```yaml
bootstrapDns:
upstream: tcp-tls:dns.example.com
ips:
- 123.123.123.123
- upstream: tcp-tls:dns.example.com
ips:
- 123.123.123.123
- upstream: https://dns2.example.com/dns-query
ips:
- 234.234.234.234
```
## Filtering

View File

@ -29,8 +29,7 @@ type Bootstrap struct {
log *logrus.Entry
resolver Resolver
upstream Resolver // the upstream that's part of the above resolver
upstreamIPs []net.IP // IPs for b.upstream
bootstraped bootstrapedResolvers
systemResolver *net.Resolver
}
@ -38,47 +37,38 @@ type Bootstrap struct {
// NewBootstrap creates and returns a new Bootstrap.
// Internally, it uses a CachingResolver and an UpstreamResolver.
func NewBootstrap(cfg *config.Config) (b *Bootstrap, err error) {
upstream := cfg.BootstrapDNS.Upstream
log := log.PrefixedLog("bootstrap")
var ips []net.IP
switch {
case upstream.IsDefault():
log.Infof("bootstrapDns is not configured, will use system resolver")
case upstream.Net == config.NetProtocolTcpUdp:
ip := net.ParseIP(upstream.Host)
if ip == nil {
return nil, fmt.Errorf("bootstrapDns uses %s but is not an IP", upstream.Net)
}
ips = append(ips, ip)
default:
ips = cfg.BootstrapDNS.IPs
if len(ips) == 0 {
return nil, fmt.Errorf("bootstrapDns.IPs is required when upstream uses %s", upstream.Net)
}
}
// Create b in multiple steps: Bootstrap and UpstreamResolver have a cyclic dependency
// This also prevents the GC to clean up these two structs, but is not currently an
// issue since they stay allocated until the process terminates
b = &Bootstrap{
log: log,
upstreamIPs: ips,
systemResolver: net.DefaultResolver, // allow replacing it during tests
}
if upstream.IsDefault() {
bootstraped, err := newBootstrapedResolvers(b, cfg.BootstrapDNS)
if err != nil {
return nil, err
}
if len(bootstraped) == 0 {
log.Infof("bootstrapDns is not configured, will use system resolver")
return b, nil
}
b.upstream = newUpstreamResolverUnchecked(upstream, b)
parallelResolver, err := newParallelBestResolver(bootstraped.ResolverGroups())
if err != nil {
return nil, fmt.Errorf("could not create bootstrap ParallelBestResolver: %w", err)
}
b.bootstraped = bootstraped
b.resolver = Chain(
NewFilteringResolver(cfg.Filtering),
NewCachingResolver(cfg.Caching, nil),
b.upstream,
parallelResolver,
)
return b, nil
@ -116,9 +106,9 @@ func (b *Bootstrap) resolveUpstream(r Resolver, host string) ([]net.IP, error) {
return b.systemResolver.LookupIP(ctx, cfg.ConnectIPVersion.Net(), host)
}
if r == b.upstream {
// Special path for b.upstream to avoid infinite recursion
return b.upstreamIPs, nil
if ips, ok := b.bootstraped[r]; ok {
// Special path for bootstraped upstreams to avoid infinite recursion
return ips, nil
}
return b.resolve(host, v4v6QTypes)
@ -232,6 +222,74 @@ func (b *Bootstrap) resolveType(hostname string, qType dns.Type) (ips []net.IP,
return ips, nil
}
// map of bootstraped resolvers their hardcoded IPs
type bootstrapedResolvers map[Resolver][]net.IP
func newBootstrapedResolvers(b *Bootstrap, cfg config.BootstrapDNSConfig) (bootstrapedResolvers, error) {
upstreamIPs := make(bootstrapedResolvers, len(cfg))
var multiErr *multierror.Error
for i, upstreamCfg := range cfg {
i := i + 1 // user visible index should start at 1
upstream := upstreamCfg.Upstream
var ips []net.IP
switch {
case upstream.IsDefault():
multiErr = multierror.Append(
multiErr,
fmt.Errorf("item %d: upstream not configured (ips=%v)", i, upstreamCfg.IPs),
)
continue
case upstream.Net == config.NetProtocolTcpUdp:
ip := net.ParseIP(upstream.Host)
if ip == nil {
multiErr = multierror.Append(
multiErr,
fmt.Errorf("item %d: '%s': protocol %s must use IP instead of hostname", i, upstream, upstream.Net),
)
continue
}
ips = append(ips, ip)
default:
ips = upstreamCfg.IPs
if len(ips) == 0 {
multiErr = multierror.Append(
multiErr,
fmt.Errorf("item %d: '%s': protocol %s requires IPs to be set", i, upstream, upstream.Net),
)
continue
}
}
resolver := newUpstreamResolverUnchecked(upstream, b)
upstreamIPs[resolver] = ips
}
if multiErr != nil {
return nil, fmt.Errorf("invalid bootstrapDns configuration: %w", multiErr)
}
return upstreamIPs, nil
}
func (br bootstrapedResolvers) ResolverGroups() map[string][]Resolver {
resolvers := make([]Resolver, 0, len(br))
for resolver := range br {
resolvers = append(resolvers, resolver)
}
return map[string][]Resolver{
upstreamDefaultCfgName: resolvers,
}
}
type IPSet struct {
values []net.IP
index uint32

View File

@ -31,12 +31,14 @@ var _ = Describe("Bootstrap", Label("bootstrap"), func() {
BeforeEach(func() {
sutConfig = &config.Config{
BootstrapDNS: config.BootstrapConfig{
Upstream: config.Upstream{
Net: config.NetProtocolTcpTls,
Host: "bootstrapUpstream.invalid",
BootstrapDNS: []config.BootstrappedUpstreamConfig{
{
Upstream: config.Upstream{
Net: config.NetProtocolTcpTls,
Host: "bootstrapUpstream.invalid",
},
IPs: []net.IP{net.IPv4zero},
},
IPs: []net.IP{net.IPv4zero},
},
}
})
@ -77,58 +79,92 @@ var _ = Describe("Bootstrap", Label("bootstrap"), func() {
Expect(*transport).Should(BeZero()) //nolint:govet
})
})
})
Context("using TCP UDP", func() {
When("IP is set", func() {
BeforeEach(func() {
sutConfig = &config.Config{
BootstrapDNS: config.BootstrapConfig{
Upstream: config.Upstream{
Net: config.NetProtocolTcpUdp,
Host: "0.0.0.0",
},
},
}
})
It("accepts an IP", func() {
Expect(sut).ShouldNot(BeNil())
Expect(sut.upstreamIPs).Should(ContainElement(net.IPv4zero))
})
})
When("IP is invalid", func() {
It("requires an IP", func() {
When("one of multiple upstreams is invalid", func() {
It("errors", func() {
cfg := config.Config{
BootstrapDNS: config.BootstrapConfig{
Upstream: config.Upstream{
Net: config.NetProtocolTcpUdp,
Host: "bootstrapUpstream.invalid",
BootstrapDNS: []config.BootstrappedUpstreamConfig{
{
Upstream: config.Upstream{ // valid
Net: config.NetProtocolTcpUdp,
Host: "0.0.0.0",
},
},
{
Upstream: config.Upstream{ // invalid
Net: config.NetProtocolTcpUdp,
Host: "hostname",
},
},
},
}
_, err := NewBootstrap(&cfg)
Expect(err).ShouldNot(Succeed())
Expect(err.Error()).Should(ContainSubstring("is not an IP"))
})
})
})
Context("using TCP UDP", func() {
When("hostname is an IP", func() {
BeforeEach(func() {
sutConfig = &config.Config{
BootstrapDNS: []config.BootstrappedUpstreamConfig{
{
Upstream: config.Upstream{
Net: config.NetProtocolTcpUdp,
Host: "0.0.0.0",
},
},
},
}
})
It("uses it", func() {
Expect(sut).ShouldNot(BeNil())
for _, ips := range sut.bootstraped {
Expect(ips).Should(Equal([]net.IP{net.IPv4zero}))
}
})
})
When("IP is invalid", func() {
It("errors", func() {
cfg := config.Config{
BootstrapDNS: []config.BootstrappedUpstreamConfig{
{
Upstream: config.Upstream{
Net: config.NetProtocolTcpUdp,
Host: "bootstrapUpstream.invalid",
},
},
},
}
_, err := NewBootstrap(&cfg)
Expect(err).ShouldNot(Succeed())
Expect(err.Error()).Should(ContainSubstring("must use IP instead of hostname"))
})
})
})
Context("using encrypted DNS", func() {
When("IP is invalid", func() {
It("requires bootstrap IPs", func() {
When("IPs are missing", func() {
It("errors", func() {
cfg := config.Config{
BootstrapDNS: config.BootstrapConfig{
Upstream: config.Upstream{
Net: config.NetProtocolTcpTls,
Host: "bootstrapUpstream.invalid",
BootstrapDNS: []config.BootstrappedUpstreamConfig{
{
Upstream: config.Upstream{
Net: config.NetProtocolTcpTls,
Host: "bootstrapUpstream.invalid",
},
},
},
}
_, err := NewBootstrap(&cfg)
Expect(err).ShouldNot(Succeed())
Expect(err.Error()).Should(ContainSubstring("bootstrapDns.IPs is required"))
Expect(err.Error()).Should(ContainSubstring("requires IPs to be set"))
})
})
})
@ -140,18 +176,20 @@ var _ = Describe("Bootstrap", Label("bootstrap"), func() {
BeforeEach(func() {
bootstrapUpstream = &mockResolver{}
sutConfig.BootstrapDNS = config.BootstrapConfig{
Upstream: config.Upstream{
Net: config.NetProtocolTcpTls,
Host: "bootstrapUpstream.invalid",
sutConfig.BootstrapDNS = []config.BootstrappedUpstreamConfig{
{
Upstream: config.Upstream{
Net: config.NetProtocolTcpTls,
Host: "bootstrapUpstream.invalid",
},
IPs: []net.IP{net.IPv4zero},
},
IPs: []net.IP{net.IPv4zero},
}
})
JustBeforeEach(func() {
sut.resolver = bootstrapUpstream
sut.upstream = bootstrapUpstream
sut.bootstraped = bootstrapedResolvers{bootstrapUpstream: sutConfig.BootstrapDNS[0].IPs}
})
AfterEach(func() {
@ -163,7 +201,7 @@ var _ = Describe("Bootstrap", Label("bootstrap"), func() {
ips, err := sut.resolveUpstream(bootstrapUpstream, "host")
Expect(err).Should(Succeed())
Expect(ips).Should(Equal(sutConfig.BootstrapDNS.IPs))
Expect(ips).Should(Equal(sutConfig.BootstrapDNS[0].IPs))
})
})
@ -320,4 +358,35 @@ var _ = Describe("Bootstrap", Label("bootstrap"), func() {
})
})
})
Describe("multiple upstreams", func() {
var (
mockUpstream1 *MockUDPUpstreamServer
mockUpstream2 *MockUDPUpstreamServer
)
BeforeEach(func() {
mockUpstream1 = NewMockUDPUpstreamServer().WithAnswerRR("example.com 123 IN A 123.124.122.122")
DeferCleanup(mockUpstream1.Close)
mockUpstream2 = NewMockUDPUpstreamServer().WithAnswerRR("example.com 123 IN A 123.124.122.122")
DeferCleanup(mockUpstream1.Close)
sutConfig.BootstrapDNS = []config.BootstrappedUpstreamConfig{
{Upstream: mockUpstream1.Start()},
{Upstream: mockUpstream2.Start()},
}
})
It("uses both", func() {
_, err := sut.resolve("example.com.", []dns.Type{dns.Type(dns.TypeA)})
Expect(err).To(Succeed())
Eventually(func(g Gomega) {
g.Expect(mockUpstream1.GetCallCount()).To(Equal(1))
g.Expect(mockUpstream2.GetCallCount()).To(Equal(1))
}, "100ms").Should(Succeed())
})
})
})

View File

@ -2,6 +2,7 @@ package resolver
import (
"io"
"net"
"net/http"
"net/http/httptest"
@ -83,7 +84,7 @@ func newTestBootstrap(response *dns.Msg) *Bootstrap {
util.FatalOnError("can't create bootstrap", err)
b.resolver = bootstrapUpstream
b.upstream = bootstrapUpstream
b.bootstraped = bootstrapedResolvers{bootstrapUpstream: []net.IP{}}
if response != nil {
bootstrapUpstream.

View File

@ -80,14 +80,14 @@ func NewParallelBestResolver(
) (Resolver, error) {
logger := logger(parallelResolverLogger)
s := make(map[string][]*upstreamResolverStatus, len(upstreamResolvers))
resolverGroups := make(map[string][]Resolver, len(upstreamResolvers))
for name, upstreamCfgs := range upstreamResolvers {
group := make([]*upstreamResolverStatus, 0, len(upstreamCfgs))
group := make([]Resolver, 0, len(upstreamCfgs))
hasValidResolver := false
for _, u := range upstreamCfgs {
r, err := NewUpstreamResolver(u, bootstrap, shouldVerifyUpstreams)
resolver, err := NewUpstreamResolver(u, bootstrap, shouldVerifyUpstreams)
if err != nil {
logger.Warnf("upstream group %s: %v", name, err)
@ -95,7 +95,7 @@ func NewParallelBestResolver(
}
if shouldVerifyUpstreams {
err = testResolver(r)
err = testResolver(resolver)
if err != nil {
logger.Warn(err)
} else {
@ -103,22 +103,42 @@ func NewParallelBestResolver(
}
}
group = append(group, newUpstreamResolverStatus(r))
group = append(group, resolver)
}
if shouldVerifyUpstreams && !hasValidResolver {
return nil, fmt.Errorf("no valid upstream for group %s", name)
}
s[name] = group
resolverGroups[name] = group
}
if len(s[upstreamDefaultCfgName]) == 0 {
return newParallelBestResolver(resolverGroups)
}
func newParallelBestResolver(resolverGroups map[string][]Resolver) (Resolver, error) {
resolversPerClient := make(map[string][]*upstreamResolverStatus, len(resolverGroups))
for groupName, resolvers := range resolverGroups {
resolverStatuses := make([]*upstreamResolverStatus, 0, len(resolvers))
for _, r := range resolvers {
resolverStatuses = append(resolverStatuses, newUpstreamResolverStatus(r))
}
resolversPerClient[groupName] = resolverStatuses
}
if len(resolversPerClient[upstreamDefaultCfgName]) == 0 {
return nil, fmt.Errorf("no external DNS resolvers configured as default upstream resolvers. "+
"Please configure at least one under '%s' configuration name", upstreamDefaultCfgName)
}
return &ParallelBestResolver{resolversPerClient: s}, nil
r := ParallelBestResolver{
resolversPerClient: resolversPerClient,
}
return &r, nil
}
// Configuration returns current resolver configuration