Refactoring/e2e tests (#1316)

* WithNetwork refactoring

* removed tmpDir for blocky

* removed tmpDir from HTTPServer
This commit is contained in:
Kwitsch 2024-01-17 17:16:16 +01:00 committed by GitHub
parent 49c808f71d
commit 2d3ad83087
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 286 additions and 173 deletions

View File

@ -25,7 +25,7 @@ var _ = Describe("Basic functional tests", func() {
})
When("wrong port configuration is provided", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"upstreams:",
" groups:",
" default:",
@ -50,7 +50,7 @@ var _ = Describe("Basic functional tests", func() {
})
When("Minimal configuration is provided", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"upstreams:",
" groups:",
" default:",
@ -81,7 +81,7 @@ var _ = Describe("Basic functional tests", func() {
Context("http port configuration", func() {
When("'httpPort' is not defined", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"upstreams:",
" groups:",
" default:",
@ -101,7 +101,7 @@ var _ = Describe("Basic functional tests", func() {
})
When("'httpPort' is defined", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"upstreams:",
" groups:",
" default:",
@ -142,7 +142,7 @@ var _ = Describe("Basic functional tests", func() {
})
When("log privacy is enabled", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"upstreams:",
" groups:",
" default:",

View File

@ -21,7 +21,7 @@ var _ = Describe("External lists and query blocking", func() {
When("external blacklist ist not available", func() {
Context("loading.strategy = blocking", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",
@ -56,7 +56,7 @@ var _ = Describe("External lists and query blocking", func() {
})
Context("loading.strategy = failOnError", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",
@ -93,10 +93,10 @@ var _ = Describe("External lists and query blocking", func() {
Describe("Query blocking against external blacklists", func() {
When("external blacklists are defined and available", func() {
BeforeEach(func(ctx context.Context) {
_, err = createHTTPServerContainer(ctx, "httpserver", tmpDir, "list.txt", "blockeddomain.com")
_, err = createHTTPServerContainer(ctx, "httpserver", "list.txt", "blockeddomain.com")
Expect(err).Should(Succeed())
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",

View File

@ -1,25 +1,25 @@
package e2e
import (
"bufio"
"context"
"fmt"
"io"
"net"
"net/http"
"os"
"strings"
"time"
log "github.com/sirupsen/logrus"
"github.com/0xERR0R/blocky/config"
"github.com/0xERR0R/blocky/helpertest"
"github.com/0xERR0R/blocky/util"
"github.com/avast/retry-go/v4"
"github.com/docker/docker/api/types/container"
"github.com/docker/go-connections/nat"
"github.com/miekg/dns"
"github.com/onsi/ginkgo/v2"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/mariadb"
"github.com/testcontainers/testcontainers-go/modules/postgres"
@ -27,9 +27,7 @@ import (
"github.com/testcontainers/testcontainers-go/wait"
)
//nolint:gochecknoglobals
var NetworkName = fmt.Sprintf("blocky-e2e-network_%d", time.Now().Unix())
// container image names
const (
redisImage = "redis:7"
postgresImage = "postgres:15.2-alpine"
@ -39,18 +37,15 @@ const (
blockyImage = "blocky-e2e"
)
func deferTerminate[T testcontainers.Container](container T, err error) (T, error) {
ginkgo.DeferCleanup(func(ctx context.Context) error {
if container.IsRunning() {
return container.Terminate(ctx)
}
return nil
})
return container, err
}
// helper constants
const (
modeOwner = 700
startupTimeout = 30 * time.Second
)
// createDNSMokkaContainer creates a DNS mokka container with the given rules attached to the test network
// under the given alias.
// It is automatically terminated when the test is finished.
func createDNSMokkaContainer(ctx context.Context, alias string, rules ...string) (testcontainers.Container, error) {
mokaRules := make(map[string]string)
@ -59,66 +54,52 @@ func createDNSMokkaContainer(ctx context.Context, alias string, rules ...string)
}
req := testcontainers.ContainerRequest{
Image: mokaImage,
Networks: []string{NetworkName},
ExposedPorts: []string{"53/tcp", "53/udp"},
NetworkAliases: map[string][]string{NetworkName: {alias}},
WaitingFor: wait.ForExposedPort(),
Env: mokaRules,
Image: mokaImage,
ExposedPorts: []string{"53/tcp", "53/udp"},
WaitingFor: wait.ForExposedPort(),
Env: mokaRules,
}
return deferTerminate(testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
}))
return startContainerWithNetwork(ctx, req, alias)
}
func createHTTPServerContainer(ctx context.Context, alias string, tmpDir *helpertest.TmpFolder,
filename string, lines ...string,
// createHTTPServerContainer creates a static HTTP server container that serves one file with the given lines
// and is attached to the test network under the given alias.
// It is automatically terminated when the test is finished.
func createHTTPServerContainer(ctx context.Context, alias string, filename string, lines ...string,
) (testcontainers.Container, error) {
f1 := tmpDir.CreateStringFile(filename,
lines...,
)
const modeOwner = 700
file := createTempFile(lines...)
req := testcontainers.ContainerRequest{
Image: staticServerImage,
Networks: []string{NetworkName},
NetworkAliases: map[string][]string{NetworkName: {alias}},
Image: staticServerImage,
ExposedPorts: []string{"8080/tcp"},
Env: map[string]string{"FOLDER": "/"},
Files: []testcontainers.ContainerFile{
{
HostFilePath: f1.Path,
HostFilePath: file,
ContainerFilePath: fmt.Sprintf("/%s", filename),
FileMode: modeOwner,
},
},
}
return deferTerminate(testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
}))
}
func WithNetwork(network string) testcontainers.CustomizeRequestOption {
return func(req *testcontainers.GenericContainerRequest) {
req.NetworkAliases = map[string][]string{NetworkName: {network}}
req.Networks = []string{NetworkName}
}
return startContainerWithNetwork(ctx, req, alias)
}
// createRedisContainer creates a redis container attached to the test network under the alias 'redis'.
// It is automatically terminated when the test is finished.
func createRedisContainer(ctx context.Context) (*redis.RedisContainer, error) {
return deferTerminate(redis.RunContainer(ctx,
testcontainers.WithImage(redisImage),
redis.WithLogLevel(redis.LogLevelVerbose),
WithNetwork("redis"),
WithNetwork(ctx, "redis"),
))
}
// createPostgresContainer creates a postgres container attached to the test network under the alias 'postgres'.
// It creates a database 'user' with user 'user' and password 'user'.
// It is automatically terminated when the test is finished.
func createPostgresContainer(ctx context.Context) (*postgres.PostgresContainer, error) {
const waitLogOccurrence = 2
@ -132,46 +113,44 @@ func createPostgresContainer(ctx context.Context) (*postgres.PostgresContainer,
wait.ForLog("database system is ready to accept connections").
WithOccurrence(waitLogOccurrence).
WithStartupTimeout(startupTimeout)),
WithNetwork("postgres"),
WithNetwork(ctx, "postgres"),
))
}
// createMariaDBContainer creates a mariadb container attached to the test network under the alias 'mariaDB'.
// It creates a database 'user' with user 'user' and password 'user'.
// It is automatically terminated when the test is finished.
func createMariaDBContainer(ctx context.Context) (*mariadb.MariaDBContainer, error) {
return deferTerminate(mariadb.RunContainer(ctx,
testcontainers.WithImage(mariaDBImage),
mariadb.WithDatabase("user"),
mariadb.WithUsername("user"),
mariadb.WithPassword("user"),
WithNetwork("mariaDB"),
WithNetwork(ctx, "mariaDB"),
))
}
const (
modeOwner = 700
startupTimeout = 30 * time.Second
)
func createBlockyContainer(ctx context.Context, tmpDir *helpertest.TmpFolder,
// createBlockyContainer creates a blocky container with a config provided by the given lines.
// It is attached to the test network under the alias 'blocky'.
// It is automatically terminated when the test is finished.
func createBlockyContainer(ctx context.Context,
lines ...string,
) (testcontainers.Container, error) {
f1 := tmpDir.CreateStringFile("config1.yaml",
lines...,
)
confFile := createTempFile(lines...)
cfg, err := config.LoadConfig(f1.Path, true)
cfg, err := config.LoadConfig(confFile, true)
if err != nil {
return nil, fmt.Errorf("can't create config struct %w", err)
}
req := testcontainers.ContainerRequest{
Image: blockyImage,
Networks: []string{NetworkName},
Image: blockyImage,
ExposedPorts: []string{"53/tcp", "53/udp", "4000/tcp"},
Files: []testcontainers.ContainerFile{
{
HostFilePath: f1.Path,
HostFilePath: confFile,
ContainerFilePath: "/app/config.yml",
FileMode: modeOwner,
},
@ -184,15 +163,12 @@ func createBlockyContainer(ctx context.Context, tmpDir *helpertest.TmpFolder,
WaitingFor: wait.ForHealthCheck().WithStartupTimeout(startupTimeout),
}
container, err := deferTerminate(testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
}))
container, err := startContainerWithNetwork(ctx, req, "blocky")
if err != nil {
// attach container log if error occurs
if r, err := container.Logs(ctx); err == nil {
if b, err := io.ReadAll(r); err == nil {
ginkgo.AddReportEntry("blocky container log", string(b))
AddReportEntry("blocky container log", string(b))
}
}
@ -279,55 +255,25 @@ func doHTTPRequest(ctx context.Context, container testcontainers.Container, cont
return err
}
func doDNSRequest(ctx context.Context, container testcontainers.Container, message *dns.Msg) (*dns.Msg, error) {
const timeout = 5 * time.Second
// createTempFile creates a temporary file with the given lines which is deleted after the test
// Each created file is prefixed with 'blocky_e2e_file-'
func createTempFile(lines ...string) string {
file, err := os.CreateTemp("", "blocky_e2e_file-")
Expect(err).Should(Succeed())
c := &dns.Client{
Net: "tcp",
Timeout: timeout,
}
DeferCleanup(func() error {
return os.Remove(file.Name())
})
host, port, err := getContainerHostPort(ctx, container, "53/tcp")
if err != nil {
return nil, err
}
msg, _, err := c.Exchange(message, net.JoinHostPort(host, port))
return msg, err
}
func getContainerHostPort(ctx context.Context, c testcontainers.Container, p nat.Port) (host, port string, err error) {
res, err := c.MappedPort(ctx, p)
if err != nil {
return "", "", err
}
host, err = c.Host(ctx)
if err != nil {
return "", "", err
}
return host, res.Port(), err
}
func getContainerLogs(ctx context.Context, c testcontainers.Container) (lines []string, err error) {
if r, err := c.Logs(ctx); err == nil {
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
if len(strings.TrimSpace(line)) > 0 {
lines = append(lines, line)
}
for i, l := range lines {
if i != 0 {
_, err := file.WriteString("\n")
Expect(err).Should(Succeed())
}
if err := scanner.Err(); err != nil {
return nil, err
}
return lines, nil
_, err := file.WriteString(l)
Expect(err).Should(Succeed())
}
return nil, err
return file.Name()
}

View File

@ -5,13 +5,9 @@ import (
"testing"
"time"
"github.com/0xERR0R/blocky/helpertest"
"github.com/avast/retry-go/v4"
"github.com/0xERR0R/blocky/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/testcontainers/testcontainers-go"
)
func init() {
@ -23,36 +19,6 @@ func TestLists(t *testing.T) {
RunSpecs(t, "e2e Suite", Label("e2e"))
}
var (
network testcontainers.Network
tmpDir *helpertest.TmpFolder
)
var _ = BeforeSuite(func(ctx context.Context) {
var err error
network, err = testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{
NetworkRequest: testcontainers.NetworkRequest{
Name: NetworkName,
CheckDuplicate: false,
Attachable: true,
},
})
Expect(err).Should(Succeed())
DeferCleanup(func(ctx context.Context) {
err := retry.Do(
func() error {
return network.Remove(ctx)
},
retry.Attempts(3),
retry.DelayType(retry.BackOffDelay),
retry.Delay(time.Second))
Expect(err).Should(Succeed())
})
tmpDir = helpertest.NewTmpFolder("config")
SetDefaultEventuallyTimeout(5 * time.Second)
})

149
e2e/helper.go Normal file
View File

@ -0,0 +1,149 @@
package e2e
import (
"bufio"
"context"
"fmt"
"net"
"strings"
"time"
"github.com/docker/go-connections/nat"
"github.com/miekg/dns"
"github.com/onsi/ginkgo/v2"
"github.com/testcontainers/testcontainers-go"
)
//nolint:gochecknoglobals
var (
// currentNetwork is the global test network instance.
currentNetwork = testNetwork{}
)
// WithNetwork attaches the container with the given alias to the test network
func WithNetwork(ctx context.Context, alias string) testcontainers.CustomizeRequestOption {
return func(req *testcontainers.GenericContainerRequest) {
networkName := currentNetwork.Name()
network, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{
NetworkRequest: testcontainers.NetworkRequest{
Name: networkName,
CheckDuplicate: true, // force the Docker provider to reuse an existing network
Attachable: true,
},
})
if err != nil && !strings.Contains(err.Error(), "already exists") {
ginkgo.Fail(fmt.Sprintf("Failed to create network '%s'. Container won't be attached to this network: %v",
networkName, err))
return
}
// decrement the network counter when the test is finished and remove the network if it is not used anymore.
ginkgo.DeferCleanup(func(ctx context.Context) error {
if currentNetwork.Detach() {
if err := network.Remove(ctx); err != nil &&
!strings.Contains(err.Error(), "removing") &&
!strings.Contains(err.Error(), "not found") {
return err
}
}
return nil
})
// increment the network counter when the container is created.
currentNetwork.Attach()
// attaching to the network because it was created with success or it already existed.
req.Networks = append(req.Networks, networkName)
if req.NetworkAliases == nil {
req.NetworkAliases = make(map[string][]string)
}
req.NetworkAliases[networkName] = []string{alias}
}
}
// deferTerminate is a helper function to terminate the container when the test is finished.
func deferTerminate[T testcontainers.Container](container T, err error) (T, error) {
ginkgo.DeferCleanup(func(ctx context.Context) error {
if container.IsRunning() {
return container.Terminate(ctx)
}
return nil
})
return container, err
}
// startContainerWithNetwork starts the container with the given alias and attaches it to the test network.
// The container is wrapped with deferTerminate to terminate the container when the test is finished.
func startContainerWithNetwork(ctx context.Context, req testcontainers.ContainerRequest, alias string,
) (testcontainers.Container, error) {
greq := testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
}
WithNetwork(ctx, alias).Customize(&greq)
return deferTerminate(testcontainers.GenericContainer(ctx, greq))
}
// doDNSRequest sends the given DNS message to the container and returns the response.
func doDNSRequest(ctx context.Context, container testcontainers.Container, message *dns.Msg) (*dns.Msg, error) {
const timeout = 5 * time.Second
c := &dns.Client{
Net: "tcp",
Timeout: timeout,
}
host, port, err := getContainerHostPort(ctx, container, "53/tcp")
if err != nil {
return nil, err
}
msg, _, err := c.Exchange(message, net.JoinHostPort(host, port))
return msg, err
}
// getContainerHostPort returns the host and port of the given container and port.
func getContainerHostPort(ctx context.Context, c testcontainers.Container, p nat.Port) (host, port string, err error) {
res, err := c.MappedPort(ctx, p)
if err != nil {
return "", "", err
}
host, err = c.Host(ctx)
if err != nil {
return "", "", err
}
return host, res.Port(), err
}
// getContainerLogs returns the logs of the given container.
func getContainerLogs(ctx context.Context, c testcontainers.Container) (lines []string, err error) {
if r, err := c.Logs(ctx); err == nil {
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
if len(strings.TrimSpace(line)) > 0 {
lines = append(lines, line)
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
return lines, nil
}
return nil, err
}

View File

@ -25,17 +25,17 @@ var _ = Describe("Metrics functional tests", func() {
_, err = createDNSMokkaContainer(ctx, "moka1", `A google/NOERROR("A 1.2.3.4 123")`)
Expect(err).Should(Succeed())
_, err = createHTTPServerContainer(ctx, "httpserver1", tmpDir, "list1.txt", "domain1.com")
_, err = createHTTPServerContainer(ctx, "httpserver1", "list1.txt", "domain1.com")
Expect(err).Should(Succeed())
_, err = createHTTPServerContainer(ctx, "httpserver2", tmpDir, "list2.txt",
_, err = createHTTPServerContainer(ctx, "httpserver2", "list2.txt",
"domain1.com", "domain2", "domain3")
Expect(err).Should(Succeed())
_, err = createHTTPServerContainer(ctx, "httpserver2", tmpDir, "list2.txt", "domain1.com", "domain2", "domain3")
_, err = createHTTPServerContainer(ctx, "httpserver2", "list2.txt", "domain1.com", "domain2", "domain3")
Expect(err).Should(Succeed())
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"upstreams:",
" groups:",
" default:",

View File

@ -32,7 +32,7 @@ var _ = Describe("Query logs functional tests", func() {
mariaDB, err = createMariaDBContainer(ctx)
Expect(err).Should(Succeed())
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",
@ -107,7 +107,7 @@ var _ = Describe("Query logs functional tests", func() {
postgresDB, err = createPostgresContainer(ctx)
Expect(err).Should(Succeed())
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",

View File

@ -39,7 +39,7 @@ var _ = Describe("Redis configuration tests", func() {
Describe("Cache sharing between blocky instances", func() {
When("Redis and 2 blocky instances are configured", func() {
BeforeEach(func(ctx context.Context) {
blocky1, err = createBlockyContainer(ctx, tmpDir,
blocky1, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",
@ -51,7 +51,7 @@ var _ = Describe("Redis configuration tests", func() {
)
Expect(err).Should(Succeed())
blocky2, err = createBlockyContainer(ctx, tmpDir,
blocky2, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",
@ -102,7 +102,7 @@ var _ = Describe("Redis configuration tests", func() {
Describe("Cache loading on startup", func() {
When("Redis and 1 blocky instance are configured", func() {
BeforeEach(func(ctx context.Context) {
blocky1, err = createBlockyContainer(ctx, tmpDir,
blocky1, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",
@ -130,7 +130,7 @@ var _ = Describe("Redis configuration tests", func() {
})
By("start other instance of blocky now -> it should load the cache from redis", func() {
blocky2, err = createBlockyContainer(ctx, tmpDir,
blocky2, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",

52
e2e/testNetwork.go Normal file
View File

@ -0,0 +1,52 @@
package e2e
import (
"fmt"
"sync/atomic"
"time"
)
// testNetwork is a helper struct to create a unique network name and count the number of attached containers.
type testNetwork struct {
name atomic.Value
counter atomic.Int32
}
// Name returns the name of the test network.
func (n *testNetwork) Name() string {
if v := n.name.Load(); v != nil {
return v.(string)
}
n.Reset()
return n.Name()
}
// Reset generates a new network name.
func (n *testNetwork) Reset() {
n.name.Store(fmt.Sprintf("blocky-e2e-network_%d", time.Now().Unix()))
}
// Attach increments the network counter.
func (n *testNetwork) Attach() {
n.counter.Add(1)
}
// Detach decrements the network counter and returns true if the counter hits zero which indicates that the
// network can be removed.
func (n *testNetwork) Detach() bool {
if n.counter.Load() <= 0 {
return false
}
n.counter.Add(-1)
if n.counter.Load() == 0 {
n.Reset()
return true
}
return false
}

View File

@ -18,7 +18,7 @@ var _ = Describe("Upstream resolver configuration tests", func() {
Describe("'upstreams.init.strategy' parameter handling", func() {
When("'upstreams.init.strategy' is fast and upstream server as IP is not reachable", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",
@ -39,7 +39,7 @@ var _ = Describe("Upstream resolver configuration tests", func() {
})
When("'upstreams.init.strategy' is fast and upstream server as host name is not reachable", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"log:",
" level: warn",
"upstreams:",
@ -58,7 +58,7 @@ var _ = Describe("Upstream resolver configuration tests", func() {
})
When("'upstreams.init.strategy' is failOnError and upstream as IP address server is not reachable", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"upstreams:",
" groups:",
" default:",
@ -76,7 +76,7 @@ var _ = Describe("Upstream resolver configuration tests", func() {
})
When("'upstreams.init.strategy' is failOnError and upstream server as host name is not reachable", func() {
BeforeEach(func(ctx context.Context) {
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"upstreams:",
" groups:",
" default:",
@ -100,7 +100,7 @@ var _ = Describe("Upstream resolver configuration tests", func() {
`A delay.com/delay(NOERROR("A 1.1.1.1 100"), "300ms")`)
Expect(err).Should(Succeed())
blocky, err = createBlockyContainer(ctx, tmpDir,
blocky, err = createBlockyContainer(ctx,
"upstreams:",
" groups:",
" default:",