blocky/lists/downloader.go

package lists

import (
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"time"

	"github.com/0xERR0R/blocky/evt"
	"github.com/avast/retry-go/v4"
)

const (
	defaultDownloadTimeout  = time.Second
	defaultDownloadAttempts = uint(1)
	defaultDownloadCooldown = 500 * time.Millisecond
)

// TransientError represents a temporary error like timeout, network errors...
type TransientError struct {
	inner error
}

func (e *TransientError) Error() string {
	return fmt.Sprintf("temporary error occurred: %v", e.inner)
}

func (e *TransientError) Unwrap() error {
	return e.inner
}

// FileDownloader is able to download some text file
type FileDownloader interface {
	DownloadFile(link string) (io.ReadCloser, error)
}

// HTTPDownloader downloads files via HTTP protocol
type HTTPDownloader struct {
	downloadTimeout  time.Duration
	downloadAttempts uint
	downloadCooldown time.Duration
	httpTransport    *http.Transport
}

type DownloaderOption func(c *HTTPDownloader)

func NewDownloader(options ...DownloaderOption) *HTTPDownloader {
	d := &HTTPDownloader{
		downloadTimeout:  defaultDownloadTimeout,
		downloadAttempts: defaultDownloadAttempts,
		downloadCooldown: defaultDownloadCooldown,
		httpTransport:    &http.Transport{},
	}

	for _, opt := range options {
		opt(d)
	}

	return d
}

// WithTimeout sets the download timeout
func WithTimeout(timeout time.Duration) DownloaderOption {
	return func(d *HTTPDownloader) {
		d.downloadTimeout = timeout
	}
}

// WithTimeout sets the pause between 2 download attempts
func WithCooldown(cooldown time.Duration) DownloaderOption {
	return func(d *HTTPDownloader) {
		d.downloadCooldown = cooldown
	}
}

// WithTimeout sets the attempt number for retry
func WithAttempts(downloadAttempts uint) DownloaderOption {
	return func(d *HTTPDownloader) {
		d.downloadAttempts = downloadAttempts
	}
}

// WithTimeout sets the HTTP transport
func WithTransport(httpTransport *http.Transport) DownloaderOption {
	return func(d *HTTPDownloader) {
		d.httpTransport = httpTransport
	}
}

func (d *HTTPDownloader) DownloadFile(link string) (io.ReadCloser, error) {
	client := http.Client{
		Timeout:   d.downloadTimeout,
		Transport: d.httpTransport,
	}

	logger().WithField("link", link).Info("starting download")

	var body io.ReadCloser

	err := retry.Do(
		func() error {
			var resp *http.Response
			var httpErr error
			if resp, httpErr = client.Get(link); httpErr == nil {
				if resp.StatusCode == http.StatusOK {
					body = resp.Body

					return nil
				}

				_ = resp.Body.Close()

				return fmt.Errorf("got status code %d", resp.StatusCode)
			}
			var netErr net.Error
			if errors.As(httpErr, &netErr) && netErr.Timeout() {
				return &TransientError{inner: netErr}
			}

			return httpErr
		},
		retry.Attempts(d.downloadAttempts),
		retry.DelayType(retry.FixedDelay),
		retry.Delay(d.downloadCooldown),
		retry.LastErrorOnly(true),
		retry.OnRetry(func(n uint, err error) {
			var transientErr *TransientError

			var dnsErr *net.DNSError

			logger := logger().WithField("link", link).WithField("attempt",
				fmt.Sprintf("%d/%d", n+1, d.downloadAttempts))

			switch {
			case errors.As(err, &transientErr):
				logger.Warnf("Temporary network err / Timeout occurred: %s", transientErr)
			case errors.As(err, &dnsErr):
				logger.Warnf("Name resolution err: %s", dnsErr.Err)
			default:
				logger.Warnf("Can't download file: %s", err)
			}

			onDownloadError(link)
		}))

	return body, err
}

func onDownloadError(link string) {
	evt.Bus().Publish(evt.CachingFailedDownloadChanged, link)
}