Unit test refactoring & devcontainer enhancement(#1245)

* Whietlist dot imports for test packages

* improved generate-lcov script

* moved mock_call_sequence to helpertest

* removed default config for markdown

* generate seperate lcov.info for each directory

* path fix in lcov generation script

* ginkgo -> v2.13.1

* fixed redundant strings

* added exampleComKey

* added folder name to finish message

* hide lcov.info in devcontainer

* coverage plugin is bugged if lcov file is excluded

* moved all lcov files to coverage folder

* added requested toos #1251

.devcontainer/devcontainer.json

@@ -7,7 +7,11 @@
     "ghcr.io/devcontainers/features/docker-in-docker:2": {
       "dockerDashComposeVersion": "v2"
     },
-    "ghcr.io/devcontainers/features/python:1": {}
+    "ghcr.io/devcontainers/features/python:1": {},
+    "ghcr.io/devcontainers/features/github-cli:1": {},
+    "ghcr.io/rocker-org/devcontainer-features/apt-packages:1": {
+      "packages": "dnsutils "
+    }
   },
   "remoteEnv": {
     "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}",
@@ -27,7 +31,9 @@
         "GitHub.vscode-github-actions"
       ],
       "settings": {
-        "go.lintFlags": ["--config=${containerWorkspaceFolder}/.golangci.yml"],
+        "go.lintFlags": [
+          "--config=${containerWorkspaceFolder}/.golangci.yml"
+        ],
         "go.alternateTools": {
           "go-langserver": "gopls"
         },
@@ -39,6 +45,15 @@
         },
         "[markdown]": {
           "editor.defaultFormatter": "yzhang.markdown-all-in-one"
+        },
+        "markiscodecoverage.searchCriteria": "**/*.lcov",
+        "runItOn": {
+          "commands": [
+            {
+              "match": "\\.go$",
+              "cmd": "${workspaceRoot}/.devcontainer/scripts/runItOnGo.sh ${fileDirname} ${workspaceRoot}"
+            }
+          ]
         }
       }
     }

.devcontainer/scripts/generate-lcov.sh

@@ -1,5 +0,0 @@
-#!/bin/bash -e
-
-cd "${WORKSPACE_FOLDER}"
-
-nohup bash -c 'ginkgo --label-filter="!e2e" --no-color --keep-going --timeout=5m --coverprofile=lcov.work --covermode=set --cover -r -p && gcov2lcov -infile=coverage.txt -outfile=lcov.info' > lcov.log 2>&1

.devcontainer/scripts/runItOnGo.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -e
+
+FOLDER_PATH=$1
+if [ -z "${FOLDER_PATH}" ]; then
+    FOLDER_PATH=$PWD
+fi
+
+BASE_PATH=$2
+if [ -z "${BASE_PATH}" ]; then
+    BASE_PATH=$WORKSPACE_FOLDER
+fi
+
+if [ "$FOLDER_PATH" = "$BASE_PATH" ]; then
+    echo "Skipping lcov creation for base path"
+    exit 1
+fi
+
+FOLDER_NAME=${FOLDER_PATH#"$BASE_PATH/"}
+WORK_NAME="$(echo "$FOLDER_NAME" | sed 's/\//-/g')"
+WORK_FILE_NAME="$WORK_NAME.ginkgo"
+WORK_FILE_PATH="/tmp/$WORK_FILE_NAME"
+OUTPUT_FOLDER="$BASE_PATH/coverage"
+OUTPUT_FILE_PATH="$OUTPUT_FOLDER/$WORK_NAME.lcov"
+
+
+mkdir -p "$OUTPUT_FOLDER"
+
+echo "-- Start $FOLDER_NAME ($(date '+%T')) --"
+
+TIMEFORMAT=' - Ginkgo tests finished in: %R seconds'
+time ginkgo --label-filter="!e2e" --keep-going --timeout=5m --output-dir=/tmp --coverprofile="$WORK_FILE_NAME" --covermode=atomic --cover -r -p "$FOLDER_PATH" || true
+
+TIMEFORMAT=' - lcov convert finished in: %R seconds'
+time gcov2lcov -infile="$WORK_FILE_PATH" -outfile="$OUTPUT_FILE_PATH" || true
+
+TIMEFORMAT=' - cleanup finished in: %R seconds'
+time rm "$WORK_FILE_PATH" || true
+
+echo "-- Finished $FOLDER_NAME ($(date '+%T')) --"

.dockerignore

@@ -12,3 +12,5 @@ LICENSE
 vendor
 e2e/
 .devcontainer/
+coverage.txt
+coverage/

.gitignore

@@ -13,4 +13,4 @@ node_modules
 package-lock.json
 vendor/
 coverage.txt
-lcov.*
+coverage/

.golangci.yml

@@ -91,6 +91,12 @@ linters-settings:
     forbid-focus-container: true
     # Don't trigger warnings for HaveLen(0)
     allow-havelen-zero: true
+  stylecheck:
+    # Whietlist dot imports for test packages.
+    dot-import-whitelist:
+      - "github.com/onsi/ginkgo"
+      - "github.com/onsi/gomega"
+      - "github.com/0xERR0R/blocky/helpertest"
 
 issues:
   exclude-rules:

.vscode/settings.json

@@ -18,14 +18,5 @@
     "ui.semanticTokens": true,
     "formatting.gofumpt": true,
     "build.standaloneTags": ["ignore", "tools"]
-  },
-  "runItOn": {
-    "commands": [
-      {
-        "match": "\\.go$",
-        "cmd": "./.devcontainer/scripts/generate-lcov.sh",
-        "silent": true
-      }
-    ]
   }
 }

helpertest/mock_call_sequence.go

@@ -1,4 +1,4 @@
-package util
+package helpertest
 
 import (
 	"context"

lists/list_cache_test.go

@@ -14,7 +14,6 @@ import (
 	. "github.com/0xERR0R/blocky/evt"
 	"github.com/0xERR0R/blocky/lists/parsers"
 	"github.com/0xERR0R/blocky/log"
-	"github.com/0xERR0R/blocky/util"
 	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 
@@ -449,11 +448,11 @@ var _ = Describe("ListCache", func() {
 })
 
 type MockDownloader struct {
-	util.MockCallSequence[string]
+	MockCallSequence[string]
 }
 
 func newMockDownloader(driver func(res chan<- string, err chan<- error)) *MockDownloader {
-	return &MockDownloader{util.NewMockCallSequence(driver)}
+	return &MockDownloader{NewMockCallSequence(driver)}
 }
 
 func (m *MockDownloader) DownloadFile(_ string) (io.ReadCloser, error) {

lists/parsers/parser_test.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/0xERR0R/blocky/util"
+	. "github.com/0xERR0R/blocky/helpertest"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -163,10 +163,10 @@ func iteratorToList[T any](forEach func(func(T) error) error) []T {
 	return res
 }
 
-type mockParser[T any] struct{ util.MockCallSequence[T] }
+type mockParser[T any] struct{ MockCallSequence[T] }
 
 func newMockParser[T any](driver func(chan<- T, chan<- error)) SeriesParser[T] {
-	return &mockParser[T]{util.NewMockCallSequence(driver)}
+	return &mockParser[T]{NewMockCallSequence(driver)}
 }
 
 func (m *mockParser[T]) Next(ctx context.Context) (_ T, rerr error) {

redis/redis_test.go

@@ -14,6 +14,10 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+const (
+	exampleComKey = CacheStorePrefix + "example.com"
+)
+
 var (
 	redisServer *miniredis.Miniredis
 	redisClient *Client
@@ -100,10 +104,10 @@ var _ = Describe("Redis client", func() {
 
 				By("Database has one entry with correct TTL", func() {
 					Eventually(func() bool {
-						return redisServer.DB(redisConfig.Database).Exists(CacheStorePrefix + "example.com")
+						return redisServer.DB(redisConfig.Database).Exists(exampleComKey)
 					}).Should(BeTrue())
 
-					ttl := redisServer.DB(redisConfig.Database).TTL(CacheStorePrefix + "example.com")
+					ttl := redisServer.DB(redisConfig.Database).TTL(exampleComKey)
 					Expect(ttl.Seconds()).Should(BeNumerically("~", 123))
 				})
 			})
@@ -125,10 +129,10 @@ var _ = Describe("Redis client", func() {
 
 				By("Database has one entry with default TTL", func() {
 					Eventually(func() bool {
-						return redisServer.DB(redisConfig.Database).Exists(CacheStorePrefix + "example.com")
+						return redisServer.DB(redisConfig.Database).Exists(exampleComKey)
 					}).Should(BeTrue())
 
-					ttl := redisServer.DB(redisConfig.Database).TTL(CacheStorePrefix + "example.com")
+					ttl := redisServer.DB(redisConfig.Database).TTL(exampleComKey)
 					Expect(ttl.Seconds()).Should(BeNumerically("~", defaultCacheTime.Seconds()))
 				})
 			})

server/server_test.go

@@ -39,11 +39,13 @@ var (
 	sut                                                          *Server
 	err                                                          error
 	baseURL                                                      string
+	queryURL                                                     string
 	googleMockUpstream, fritzboxMockUpstream, clientMockUpstream *resolver.MockUDPUpstreamServer
 )
 
 var _ = BeforeSuite(func() {
 	baseURL = "http://localhost:" + GetStringPort(httpBasePort) + "/"
+	queryURL = baseURL + "dns-query"
 	var upstreamGoogle, upstreamFritzbox, upstreamClient config.Upstream
 	ctx, cancelFn := context.WithCancel(context.Background())
 	DeferCleanup(cancelFn)
@@ -415,7 +417,7 @@ var _ = Describe("Running DNS server", func() {
 		Context("DOH over GET (RFC 8484)", func() {
 			When("DOH get request with 'example.com' is performed", func() {
 				It("should get a valid response", func() {
-					resp, err := http.Get(baseURL + "dns-query?dns=AAABAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB")
+					resp, err := http.Get(queryURL + "?dns=AAABAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB")
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)
 
@@ -434,7 +436,7 @@ var _ = Describe("Running DNS server", func() {
 			})
 			When("Request does not contain a valid DNS message", func() {
 				It("should return 'Bad Request'", func() {
-					resp, err := http.Get(baseURL + "dns-query?dns=xxxx")
+					resp, err := http.Get(queryURL + "?dns=xxxx")
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)
 
@@ -443,7 +445,7 @@ var _ = Describe("Running DNS server", func() {
 			})
 			When("Request's parameter does not contain a valid base64'", func() {
 				It("should return 'Bad Request'", func() {
-					resp, err := http.Get(baseURL + "dns-query?dns=äöä")
+					resp, err := http.Get(queryURL + "?dns=äöä")
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)
 
@@ -452,7 +454,7 @@ var _ = Describe("Running DNS server", func() {
 			})
 			When("Request does not contain a dns parameter", func() {
 				It("should return 'Bad Request'", func() {
-					resp, err := http.Get(baseURL + "dns-query?test")
+					resp, err := http.Get(queryURL + "?test")
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)
 
@@ -463,7 +465,7 @@ var _ = Describe("Running DNS server", func() {
 				It("should return 'URI Too Long'", func() {
 					longBase64msg := base64.StdEncoding.EncodeToString([]byte(strings.Repeat("t", 513)))
 
-					resp, err := http.Get(baseURL + "dns-query?dns=" + longBase64msg)
+					resp, err := http.Get(queryURL + "?dns=" + longBase64msg)
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)
 
@@ -482,7 +484,7 @@ var _ = Describe("Running DNS server", func() {
 					rawDNSMessage, err := msg.Pack()
 					Expect(err).Should(Succeed())
 
-					resp, err = http.Post(baseURL+"dns-query",
+					resp, err = http.Post(queryURL,
 						"application/dns-message", bytes.NewReader(rawDNSMessage))
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)
@@ -507,7 +509,7 @@ var _ = Describe("Running DNS server", func() {
 					rawDNSMessage, err := msg.Pack()
 					Expect(err).Should(Succeed())
 
-					resp, err = http.Post(baseURL+"dns-query/client123",
+					resp, err = http.Post(queryURL+"/client123",
 						"application/dns-message", bytes.NewReader(rawDNSMessage))
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)
@@ -531,7 +533,7 @@ var _ = Describe("Running DNS server", func() {
 				It("should return 'Payload Too Large'", func() {
 					largeMessage := []byte(strings.Repeat("t", 513))
 
-					resp, err = http.Post(baseURL+"dns-query", "application/dns-message", bytes.NewReader(largeMessage))
+					resp, err = http.Post(queryURL, "application/dns-message", bytes.NewReader(largeMessage))
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)
 
@@ -540,7 +542,7 @@ var _ = Describe("Running DNS server", func() {
 			})
 			When("Request has wrong type", func() {
 				It("should return 'Unsupported Media Type'", func() {
-					resp, err = http.Post(baseURL+"dns-query", "application/text", bytes.NewReader([]byte("a")))
+					resp, err = http.Post(queryURL, "application/text", bytes.NewReader([]byte("a")))
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)
 
@@ -553,7 +555,7 @@ var _ = Describe("Running DNS server", func() {
 					rawDNSMessage, err := msg.Pack()
 					Expect(err).Should(Succeed())
 
-					resp, err = http.Post(baseURL+"dns-query",
+					resp, err = http.Post(queryURL,
 						"application/dns-message", bytes.NewReader(rawDNSMessage))
 					Expect(err).Should(Succeed())
 					DeferCleanup(resp.Body.Close)