From 6c000090b16fb698d56e0775cfa4f3c67e450862 Mon Sep 17 00:00:00 2001 From: Kwitsch Date: Wed, 15 Nov 2023 16:42:53 +0100 Subject: [PATCH] Unit test refactoring & devcontainer enhancement(#1245) * Whietlist dot imports for test packages * improved generate-lcov script * moved mock_call_sequence to helpertest * removed default config for markdown * generate seperate lcov.info for each directory * path fix in lcov generation script * ginkgo -> v2.13.1 * fixed redundant strings * added exampleComKey * added folder name to finish message * hide lcov.info in devcontainer * coverage plugin is bugged if lcov file is excluded * moved all lcov files to coverage folder * added requested toos #1251 --- .devcontainer/devcontainer.json | 19 +++++++++-- .devcontainer/scripts/generate-lcov.sh | 5 --- .devcontainer/scripts/runItOnGo.sh | 39 ++++++++++++++++++++++ .dockerignore | 2 ++ .gitignore | 2 +- .golangci.yml | 6 ++++ .vscode/settings.json | 9 ----- {util => helpertest}/mock_call_sequence.go | 2 +- lists/list_cache_test.go | 5 ++- lists/parsers/parser_test.go | 6 ++-- redis/redis_test.go | 12 ++++--- server/server_test.go | 22 ++++++------ 12 files changed, 91 insertions(+), 38 deletions(-) delete mode 100644 .devcontainer/scripts/generate-lcov.sh create mode 100644 .devcontainer/scripts/runItOnGo.sh rename {util => helpertest}/mock_call_sequence.go (98%) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2b0da1d9..ec941e14 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,7 +7,11 @@ "ghcr.io/devcontainers/features/docker-in-docker:2": { "dockerDashComposeVersion": "v2" }, - "ghcr.io/devcontainers/features/python:1": {} + "ghcr.io/devcontainers/features/python:1": {}, + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/rocker-org/devcontainer-features/apt-packages:1": { + "packages": "dnsutils " + } }, "remoteEnv": { "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}", @@ -27,7 +31,9 @@ "GitHub.vscode-github-actions" ], "settings": { - "go.lintFlags": ["--config=${containerWorkspaceFolder}/.golangci.yml"], + "go.lintFlags": [ + "--config=${containerWorkspaceFolder}/.golangci.yml" + ], "go.alternateTools": { "go-langserver": "gopls" }, @@ -39,6 +45,15 @@ }, "[markdown]": { "editor.defaultFormatter": "yzhang.markdown-all-in-one" + }, + "markiscodecoverage.searchCriteria": "**/*.lcov", + "runItOn": { + "commands": [ + { + "match": "\\.go$", + "cmd": "${workspaceRoot}/.devcontainer/scripts/runItOnGo.sh ${fileDirname} ${workspaceRoot}" + } + ] } } } diff --git a/.devcontainer/scripts/generate-lcov.sh b/.devcontainer/scripts/generate-lcov.sh deleted file mode 100644 index 7e7675ef..00000000 --- a/.devcontainer/scripts/generate-lcov.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -e - -cd "${WORKSPACE_FOLDER}" - -nohup bash -c 'ginkgo --label-filter="!e2e" --no-color --keep-going --timeout=5m --coverprofile=lcov.work --covermode=set --cover -r -p && gcov2lcov -infile=coverage.txt -outfile=lcov.info' > lcov.log 2>&1 \ No newline at end of file diff --git a/.devcontainer/scripts/runItOnGo.sh b/.devcontainer/scripts/runItOnGo.sh new file mode 100644 index 00000000..ce57fe93 --- /dev/null +++ b/.devcontainer/scripts/runItOnGo.sh @@ -0,0 +1,39 @@ +#!/bin/bash -e + +FOLDER_PATH=$1 +if [ -z "${FOLDER_PATH}" ]; then + FOLDER_PATH=$PWD +fi + +BASE_PATH=$2 +if [ -z "${BASE_PATH}" ]; then + BASE_PATH=$WORKSPACE_FOLDER +fi + +if [ "$FOLDER_PATH" = "$BASE_PATH" ]; then + echo "Skipping lcov creation for base path" + exit 1 +fi + +FOLDER_NAME=${FOLDER_PATH#"$BASE_PATH/"} +WORK_NAME="$(echo "$FOLDER_NAME" | sed 's/\//-/g')" +WORK_FILE_NAME="$WORK_NAME.ginkgo" +WORK_FILE_PATH="/tmp/$WORK_FILE_NAME" +OUTPUT_FOLDER="$BASE_PATH/coverage" +OUTPUT_FILE_PATH="$OUTPUT_FOLDER/$WORK_NAME.lcov" + + +mkdir -p "$OUTPUT_FOLDER" + +echo "-- Start $FOLDER_NAME ($(date '+%T')) --" + +TIMEFORMAT=' - Ginkgo tests finished in: %R seconds' +time ginkgo --label-filter="!e2e" --keep-going --timeout=5m --output-dir=/tmp --coverprofile="$WORK_FILE_NAME" --covermode=atomic --cover -r -p "$FOLDER_PATH" || true + +TIMEFORMAT=' - lcov convert finished in: %R seconds' +time gcov2lcov -infile="$WORK_FILE_PATH" -outfile="$OUTPUT_FILE_PATH" || true + +TIMEFORMAT=' - cleanup finished in: %R seconds' +time rm "$WORK_FILE_PATH" || true + +echo "-- Finished $FOLDER_NAME ($(date '+%T')) --" \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index b46a022a..8468acd4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,3 +12,5 @@ LICENSE vendor e2e/ .devcontainer/ +coverage.txt +coverage/ diff --git a/.gitignore b/.gitignore index df8b9c97..e63218cd 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ node_modules package-lock.json vendor/ coverage.txt -lcov.* \ No newline at end of file +coverage/ diff --git a/.golangci.yml b/.golangci.yml index 200569a5..9f41c7e1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -91,6 +91,12 @@ linters-settings: forbid-focus-container: true # Don't trigger warnings for HaveLen(0) allow-havelen-zero: true + stylecheck: + # Whietlist dot imports for test packages. + dot-import-whitelist: + - "github.com/onsi/ginkgo" + - "github.com/onsi/gomega" + - "github.com/0xERR0R/blocky/helpertest" issues: exclude-rules: diff --git a/.vscode/settings.json b/.vscode/settings.json index 1f22d135..a1acd12e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,14 +18,5 @@ "ui.semanticTokens": true, "formatting.gofumpt": true, "build.standaloneTags": ["ignore", "tools"] - }, - "runItOn": { - "commands": [ - { - "match": "\\.go$", - "cmd": "./.devcontainer/scripts/generate-lcov.sh", - "silent": true - } - ] } } diff --git a/util/mock_call_sequence.go b/helpertest/mock_call_sequence.go similarity index 98% rename from util/mock_call_sequence.go rename to helpertest/mock_call_sequence.go index 7bf12234..567bc169 100644 --- a/util/mock_call_sequence.go +++ b/helpertest/mock_call_sequence.go @@ -1,4 +1,4 @@ -package util +package helpertest import ( "context" diff --git a/lists/list_cache_test.go b/lists/list_cache_test.go index e39ffbaa..b677774e 100644 --- a/lists/list_cache_test.go +++ b/lists/list_cache_test.go @@ -14,7 +14,6 @@ import ( . "github.com/0xERR0R/blocky/evt" "github.com/0xERR0R/blocky/lists/parsers" "github.com/0xERR0R/blocky/log" - "github.com/0xERR0R/blocky/util" "github.com/google/uuid" "github.com/sirupsen/logrus" @@ -449,11 +448,11 @@ var _ = Describe("ListCache", func() { }) type MockDownloader struct { - util.MockCallSequence[string] + MockCallSequence[string] } func newMockDownloader(driver func(res chan<- string, err chan<- error)) *MockDownloader { - return &MockDownloader{util.NewMockCallSequence(driver)} + return &MockDownloader{NewMockCallSequence(driver)} } func (m *MockDownloader) DownloadFile(_ string) (io.ReadCloser, error) { diff --git a/lists/parsers/parser_test.go b/lists/parsers/parser_test.go index 25565f78..85be936a 100644 --- a/lists/parsers/parser_test.go +++ b/lists/parsers/parser_test.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/0xERR0R/blocky/util" + . "github.com/0xERR0R/blocky/helpertest" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -163,10 +163,10 @@ func iteratorToList[T any](forEach func(func(T) error) error) []T { return res } -type mockParser[T any] struct{ util.MockCallSequence[T] } +type mockParser[T any] struct{ MockCallSequence[T] } func newMockParser[T any](driver func(chan<- T, chan<- error)) SeriesParser[T] { - return &mockParser[T]{util.NewMockCallSequence(driver)} + return &mockParser[T]{NewMockCallSequence(driver)} } func (m *mockParser[T]) Next(ctx context.Context) (_ T, rerr error) { diff --git a/redis/redis_test.go b/redis/redis_test.go index 9ddf6185..04efb6b7 100644 --- a/redis/redis_test.go +++ b/redis/redis_test.go @@ -14,6 +14,10 @@ import ( . "github.com/onsi/gomega" ) +const ( + exampleComKey = CacheStorePrefix + "example.com" +) + var ( redisServer *miniredis.Miniredis redisClient *Client @@ -100,10 +104,10 @@ var _ = Describe("Redis client", func() { By("Database has one entry with correct TTL", func() { Eventually(func() bool { - return redisServer.DB(redisConfig.Database).Exists(CacheStorePrefix + "example.com") + return redisServer.DB(redisConfig.Database).Exists(exampleComKey) }).Should(BeTrue()) - ttl := redisServer.DB(redisConfig.Database).TTL(CacheStorePrefix + "example.com") + ttl := redisServer.DB(redisConfig.Database).TTL(exampleComKey) Expect(ttl.Seconds()).Should(BeNumerically("~", 123)) }) }) @@ -125,10 +129,10 @@ var _ = Describe("Redis client", func() { By("Database has one entry with default TTL", func() { Eventually(func() bool { - return redisServer.DB(redisConfig.Database).Exists(CacheStorePrefix + "example.com") + return redisServer.DB(redisConfig.Database).Exists(exampleComKey) }).Should(BeTrue()) - ttl := redisServer.DB(redisConfig.Database).TTL(CacheStorePrefix + "example.com") + ttl := redisServer.DB(redisConfig.Database).TTL(exampleComKey) Expect(ttl.Seconds()).Should(BeNumerically("~", defaultCacheTime.Seconds())) }) }) diff --git a/server/server_test.go b/server/server_test.go index 8943e191..12c27aad 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -39,11 +39,13 @@ var ( sut *Server err error baseURL string + queryURL string googleMockUpstream, fritzboxMockUpstream, clientMockUpstream *resolver.MockUDPUpstreamServer ) var _ = BeforeSuite(func() { baseURL = "http://localhost:" + GetStringPort(httpBasePort) + "/" + queryURL = baseURL + "dns-query" var upstreamGoogle, upstreamFritzbox, upstreamClient config.Upstream ctx, cancelFn := context.WithCancel(context.Background()) DeferCleanup(cancelFn) @@ -415,7 +417,7 @@ var _ = Describe("Running DNS server", func() { Context("DOH over GET (RFC 8484)", func() { When("DOH get request with 'example.com' is performed", func() { It("should get a valid response", func() { - resp, err := http.Get(baseURL + "dns-query?dns=AAABAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB") + resp, err := http.Get(queryURL + "?dns=AAABAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB") Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close) @@ -434,7 +436,7 @@ var _ = Describe("Running DNS server", func() { }) When("Request does not contain a valid DNS message", func() { It("should return 'Bad Request'", func() { - resp, err := http.Get(baseURL + "dns-query?dns=xxxx") + resp, err := http.Get(queryURL + "?dns=xxxx") Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close) @@ -443,7 +445,7 @@ var _ = Describe("Running DNS server", func() { }) When("Request's parameter does not contain a valid base64'", func() { It("should return 'Bad Request'", func() { - resp, err := http.Get(baseURL + "dns-query?dns=äöä") + resp, err := http.Get(queryURL + "?dns=äöä") Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close) @@ -452,7 +454,7 @@ var _ = Describe("Running DNS server", func() { }) When("Request does not contain a dns parameter", func() { It("should return 'Bad Request'", func() { - resp, err := http.Get(baseURL + "dns-query?test") + resp, err := http.Get(queryURL + "?test") Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close) @@ -463,7 +465,7 @@ var _ = Describe("Running DNS server", func() { It("should return 'URI Too Long'", func() { longBase64msg := base64.StdEncoding.EncodeToString([]byte(strings.Repeat("t", 513))) - resp, err := http.Get(baseURL + "dns-query?dns=" + longBase64msg) + resp, err := http.Get(queryURL + "?dns=" + longBase64msg) Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close) @@ -482,7 +484,7 @@ var _ = Describe("Running DNS server", func() { rawDNSMessage, err := msg.Pack() Expect(err).Should(Succeed()) - resp, err = http.Post(baseURL+"dns-query", + resp, err = http.Post(queryURL, "application/dns-message", bytes.NewReader(rawDNSMessage)) Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close) @@ -507,7 +509,7 @@ var _ = Describe("Running DNS server", func() { rawDNSMessage, err := msg.Pack() Expect(err).Should(Succeed()) - resp, err = http.Post(baseURL+"dns-query/client123", + resp, err = http.Post(queryURL+"/client123", "application/dns-message", bytes.NewReader(rawDNSMessage)) Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close) @@ -531,7 +533,7 @@ var _ = Describe("Running DNS server", func() { It("should return 'Payload Too Large'", func() { largeMessage := []byte(strings.Repeat("t", 513)) - resp, err = http.Post(baseURL+"dns-query", "application/dns-message", bytes.NewReader(largeMessage)) + resp, err = http.Post(queryURL, "application/dns-message", bytes.NewReader(largeMessage)) Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close) @@ -540,7 +542,7 @@ var _ = Describe("Running DNS server", func() { }) When("Request has wrong type", func() { It("should return 'Unsupported Media Type'", func() { - resp, err = http.Post(baseURL+"dns-query", "application/text", bytes.NewReader([]byte("a"))) + resp, err = http.Post(queryURL, "application/text", bytes.NewReader([]byte("a"))) Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close) @@ -553,7 +555,7 @@ var _ = Describe("Running DNS server", func() { rawDNSMessage, err := msg.Pack() Expect(err).Should(Succeed()) - resp, err = http.Post(baseURL+"dns-query", + resp, err = http.Post(queryURL, "application/dns-message", bytes.NewReader(rawDNSMessage)) Expect(err).Should(Succeed()) DeferCleanup(resp.Body.Close)