mirrors
/
blocky
mirror of https://github.com/0xERR0R/blocky.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

prefetching of often used queries

This commit is contained in:
Dimitri Herzog 2021-01-16 22:24:05 +01:00
parent 64d42e2cdd
commit e9fff3cef1
9 changed files with 842 additions and 449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/development-docker.yml vendored
View File

@ -3,6 +3,7 @@ on:
push:
branches:
- development
- fb-*
jobs:
docker:
@ -10,6 +11,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
@ -25,11 +28,16 @@ jobs:
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Extract branch name
shell: bash
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: extract_branch
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64
push: true
tags: |
ghcr.io/0xerr0r/blocky:development
spx01/blocky:development
ghcr.io/0xerr0r/blocky:${{ steps.extract_branch.outputs.branch }}
spx01/blocky:${{ steps.extract_branch.outputs.branch }}

6
.github/workflows/release.yml vendored
View File

@ -17,7 +17,10 @@ jobs:
go-version: 1.15
id: go
- uses: actions/checkout@v1
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Build
run: make tools build
@ -53,6 +56,7 @@ jobs:
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker_meta.outputs.tags }}

5
config/config.go
View File

@ -240,8 +240,9 @@ type ClientLookupConfig struct {
}
type CachingConfig struct {
MinCachingTime int `yaml:"minTime"`
MaxCachingTime int `yaml:"maxTime"`
MinCachingTime int `yaml:"minTime"`
MaxCachingTime int `yaml:"maxTime"`
Prefetching bool `yaml:"prefetching"`
}
type QueryLogConfig struct {

12
docs/README.md
View File

@ -99,14 +99,18 @@ caching:
# amount in minutes, how long a response must be cached (min value).
# If <=0, use response's TTL, if >0 use this value, if TTL is smaller
# Default: 0
minTime: 40
minTime: 5
# amount in minutes, how long a response must be cached (max value).
# If <0, do not cache responses
# If 0, use TTL
# If > 0, use this value, if TTL is greater
# Default: 0
maxTime: -1
# if true, will preload DNS results for often used queries (names queried more than 5 times in a 2 hour time window)
# this improves the response time for often used queries, but significantly increases external traffic
# default: false
prefetching: true
# optional: configuration of client name resolution
clientLookup:
# optional: this DNS resolver will be used to perform reverse DNS lookup (typically local router)
@ -114,8 +118,8 @@ clientLookup:
# optional: some routers return multiple names for client (host name and user defined name). Define which single name should be used.
# Example: take second name if present, if not take first name
singleNameOrder:
- 2
- 1
- 2
- 1
# optional: custom mapping of client name to IP addresses. Useful if reverse DNS does not work properly or just to have custom client names.
clients:
laptop:

1110
docs/blocky-grafana.json
View File

File diff suppressed because it is too large Load Diff

6
docs/config.yml
View File

@ -64,13 +64,17 @@ caching:
# amount in minutes, how long a response must be cached (min value).
# If <=0, use response's TTL, if >0 use this value, if TTL is smaller
# Default: 0
minTime: 40
minTime: 5
# amount in minutes, how long a response must be cached (max value).
# If <0, do not cache responses
# If 0, use TTL
# If > 0, use this value, if TTL is greater
# Default: 0
maxTime: -1
# if true, will preload DNS results for often used queries (names queried more than 5 times in a 2 hour time window)
# this improves the response time for often used queries, but significantly increases external traffic
# default: false
prefetching: true
# optional: configuration of client name resolution
clientLookup:

4
go.sum
View File

@ -335,6 +335,7 @@ github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNja
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw=
github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM=
github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU=
github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@ -352,6 +353,7 @@ github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lN
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.14.0 h1:RHRyE8UocrbjU+6UvRzwi6HjiDfxrrBU91TtbKzkGp4=
github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM=
github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@ -556,6 +558,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 h1:9UQO31fZ+0aKQOFldThf7BKPMJTiBfWycGh/u3UoO88=
golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -669,6 +672,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

125
resolver/caching_resolver.go
View File

@ -10,46 +10,110 @@ import (
"github.com/miekg/dns"
"github.com/patrickmn/go-cache"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
// caches answers from dns queries with their TTL time, to avoid external resolver calls for recurrent queries
type CachingResolver struct {
NextResolver
minCacheTimeSec, maxCacheTimeSec int
cachesPerType map[uint16]*cache.Cache
hitCount, missCount prometheus.Counter
entryCount prometheus.Gauge
minCacheTimeSec, maxCacheTimeSec int
cachesPerType map[uint16]*cache.Cache
prefetchingNameCache *cache.Cache
hitCount, missCount, prefetchCount prometheus.Counter
entryCount, prefetchDomainCacheCount prometheus.Gauge
}
const (
cacheTimeNegative = 30 * time.Minute
cacheTimeNegative = 30 * time.Minute
prefetchingNameCacheExpiration = 2 * time.Hour
prefetchingNameCountThreshold = 5
)
func NewCachingResolver(cfg config.CachingConfig) ChainedResolver {
var entryCount prometheus.Gauge
var entryCount, prefetchDomainCount prometheus.Gauge
var hitCount, missCount prometheus.Counter
var hitCount, missCount, prefetchCount prometheus.Counter
if metrics.IsEnabled() {
entryCount = cacheEntryCount()
prefetchDomainCount = prefetchDomainCacheCount()
hitCount = cacheHitCount()
missCount = cacheMissCount()
prefetchCount = domainPrefetchCount()
metrics.RegisterMetric(entryCount)
metrics.RegisterMetric(prefetchDomainCount)
metrics.RegisterMetric(hitCount)
metrics.RegisterMetric(missCount)
metrics.RegisterMetric(prefetchCount)
}
return &CachingResolver{
domainCache := createQueryDomainNameCache(cfg)
c := &CachingResolver{
minCacheTimeSec: 60 * cfg.MinCachingTime,
maxCacheTimeSec: 60 * cfg.MaxCachingTime,
cachesPerType: map[uint16]*cache.Cache{
dns.TypeA: cache.New(15*time.Minute, 5*time.Minute),
dns.TypeAAAA: cache.New(15*time.Minute, 5*time.Minute),
dns.TypeA: createQueryResultCache(),
dns.TypeAAAA: createQueryResultCache(),
},
entryCount: entryCount,
hitCount: hitCount,
missCount: missCount,
prefetchingNameCache: domainCache,
entryCount: entryCount,
hitCount: hitCount,
missCount: missCount,
prefetchCount: prefetchCount,
prefetchDomainCacheCount: prefetchDomainCount,
}
if cfg.Prefetching {
configurePrefetching(c)
}
return c
}
func configurePrefetching(c *CachingResolver) {
for k, v := range c.cachesPerType {
qType := k
v.OnEvicted(func(domainName string, i interface{}) {
c.onEvicted(domainName, qType)
})
}
}
func createQueryResultCache() *cache.Cache {
return cache.New(15*time.Minute, 15*time.Second)
}
func createQueryDomainNameCache(cfg config.CachingConfig) *cache.Cache {
if cfg.Prefetching {
return cache.New(prefetchingNameCacheExpiration, time.Minute)
}
return nil
}
// onEvicted is called if a DNS response in the cache is expired and was removed from cache
func (r *CachingResolver) onEvicted(domainName string, qType uint16) {
logger := logger("caching_resolver")
cnt, found := r.prefetchingNameCache.Get(domainName)
// check if domain was queried > threshold in the time window
if found && cnt.(int) > prefetchingNameCountThreshold {
logger.Debugf("prefetching '%s' (%s)", domainName, dns.TypeToString[qType])
req := newRequest(fmt.Sprintf("%s.", domainName), qType, logger)
response, err := r.next.Resolve(req)
if err == nil {
r.putInCache(response, domainName, qType)
if metrics.IsEnabled() {
r.prefetchCount.Inc()
}
} else {
logger.Errorf("can't prefetch '%s': %v", domainName, err)
}
}
}
@ -70,6 +134,14 @@ func cacheMissCount() prometheus.Counter {
},
)
}
func domainPrefetchCount() prometheus.Counter {
return prometheus.NewCounter(
prometheus.CounterOpts{
Name: "blocky_prefetch_count",
Help: "Prefetch counter",
},
)
}
func cacheEntryCount() prometheus.Gauge {
return prometheus.NewGauge(
@ -80,6 +152,15 @@ func cacheEntryCount() prometheus.Gauge {
)
}
func prefetchDomainCacheCount() prometheus.Gauge {
return prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "blocky_prefetch_domain_name_cache_count",
Help: "Number of entries in domain cache",
},
)
}
func (r *CachingResolver) getCache(queryType uint16) *cache.Cache {
return r.cachesPerType[queryType]
}
@ -94,6 +175,8 @@ func (r *CachingResolver) Configuration() (result []string) {
result = append(result, fmt.Sprintf("maxCacheTimeSec = %d", r.maxCacheTimeSec))
result = append(result, fmt.Sprintf("prefetching = %t", r.prefetchingNameCache != nil))
for t, c := range r.cachesPerType {
result = append(result, fmt.Sprintf("%s cache items count = %d", dns.TypeToString[t], c.ItemCount()))
}
@ -132,6 +215,8 @@ func (r *CachingResolver) Resolve(request *Request) (response *Response, err err
// we can cache only A and AAAA queries
if question.Qtype == dns.TypeA || question.Qtype == dns.TypeAAAA {
r.trackQueryDomainNameCount(domain, logger)
val, expiresAt, found := r.getCache(question.Qtype).GetWithExpiration(domain)
if found {
@ -179,6 +264,20 @@ func (r *CachingResolver) Resolve(request *Request) (response *Response, err err
return response, err
}
func (r *CachingResolver) trackQueryDomainNameCount(domain string, logger *logrus.Entry) {
if r.prefetchingNameCache != nil {
var domainCount int
if x, found := r.prefetchingNameCache.Get(domain); found {
domainCount = x.(int)
}
domainCount++
r.prefetchingNameCache.SetDefault(domain, domainCount)
logger.Debugf("domain '%s' was requested %d times, "+
"total cache size: %d", domain, domainCount, r.prefetchingNameCache.ItemCount())
r.prefetchDomainCacheCount.Set(float64(r.prefetchingNameCache.ItemCount()))
}
}
func (r *CachingResolver) putInCache(response *Response, domain string, qType uint16) {
answer := response.Res.Answer

11
resolver/resolver.go
View File

@ -35,10 +35,17 @@ type Request struct {
RequestTS time.Time
}
func newRequest(question string, rType uint16) *Request {
func newRequest(question string, rType uint16, logger ...*logrus.Entry) *Request {
var loggerEntry *logrus.Entry
if len(logger) == 1 {
loggerEntry = logger[0]
} else {
loggerEntry = logrus.NewEntry(logrus.New())
}
return &Request{
Req: util.NewMsgWithQuestion(question, rType),
Log: logrus.NewEntry(logrus.New()),
Log: loggerEntry,
Protocol: UDP,
}
}