mirrors
/
miniflux-v2
mirror of https://github.com/miniflux/v2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add HTTP proxy option for subscriptions

This commit is contained in:
Kebin Liu 2020-09-10 14:28:54 +08:00 committed by GitHub
parent 0f258fd55b
commit cf7712acea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 201 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/feed.go
View File

@ -51,6 +51,7 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) {
feedInfo.Password,
feedInfo.ScraperRules,
feedInfo.RewriteRules,
feedInfo.FetchViaProxy,
)
if err != nil {
json.ServerError(w, r, err)

2
api/payload.go
View File

@ -30,6 +30,7 @@ type feedCreation struct {
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
FetchViaProxy bool `json:"fetch_via_proxy"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
}
@ -39,6 +40,7 @@ type subscriptionDiscovery struct {
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
FetchViaProxy bool `json:"fetch_via_proxy"`
}
type feedModification struct {

1
api/subscription.go
View File

@ -23,6 +23,7 @@ func (h *handler) getSubscriptions(w http.ResponseWriter, r *http.Request) {
subscriptionInfo.UserAgent,
subscriptionInfo.Username,
subscriptionInfo.Password,
subscriptionInfo.FetchViaProxy,
)
if finderErr != nil {
json.ServerError(w, r, finderErr)

14
config/options.go
View File

@ -50,6 +50,7 @@ const (
defaultPocketConsumerKey = ""
defaultHTTPClientTimeout = 20
defaultHTTPClientMaxBodySize = 15
defaultHTTPClientProxy = ""
defaultAuthProxyHeader = ""
defaultAuthProxyUserCreation = false
)
@ -96,6 +97,7 @@ type Options struct {
pocketConsumerKey string
httpClientTimeout int
httpClientMaxBodySize int64
httpClientProxy string
authProxyHeader string
authProxyUserCreation bool
}
@ -141,6 +143,7 @@ func NewOptions() *Options {
pocketConsumerKey: defaultPocketConsumerKey,
httpClientTimeout: defaultHTTPClientTimeout,
httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024,
httpClientProxy: defaultHTTPClientProxy,
authProxyHeader: defaultAuthProxyHeader,
authProxyUserCreation: defaultAuthProxyUserCreation,
}
@ -349,6 +352,16 @@ func (o *Options) HTTPClientMaxBodySize() int64 {
return o.httpClientMaxBodySize
}
// HTTPClientProxy returns the proxy URL for HTTP client.
func (o *Options) HTTPClientProxy() string {
return o.httpClientProxy
}
// HasHTTPClientProxyConfigured returns true if the HTTP proxy is configured.
func (o *Options) HasHTTPClientProxyConfigured() bool {
return o.httpClientProxy != ""
}
// AuthProxyHeader returns an HTTP header name that contains username for
// authentication using auth proxy.
func (o *Options) AuthProxyHeader() string {
@ -403,6 +416,7 @@ func (o *Options) String() string {
builder.WriteString(fmt.Sprintf("OAUTH2_PROVIDER: %v\n", o.oauth2Provider))
builder.WriteString(fmt.Sprintf("HTTP_CLIENT_TIMEOUT: %v\n", o.httpClientTimeout))
builder.WriteString(fmt.Sprintf("HTTP_CLIENT_MAX_BODY_SIZE: %v\n", o.httpClientMaxBodySize))
builder.WriteString(fmt.Sprintf("HTTP_CLIENT_PROXY: %v\n", o.httpClientProxy))
builder.WriteString(fmt.Sprintf("AUTH_PROXY_HEADER: %v\n", o.authProxyHeader))
builder.WriteString(fmt.Sprintf("AUTH_PROXY_USER_CREATION: %v\n", o.authProxyUserCreation))
return builder.String()

2
config/parser.go
View File

@ -184,6 +184,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.httpClientTimeout = parseInt(value, defaultHTTPClientTimeout)
case "HTTP_CLIENT_MAX_BODY_SIZE":
p.opts.httpClientMaxBodySize = int64(parseInt(value, defaultHTTPClientMaxBodySize) * 1024 * 1024)
case "HTTP_CLIENT_PROXY":
p.opts.httpClientProxy = parseString(value, defaultHTTPClientProxy)
case "AUTH_PROXY_HEADER":
p.opts.authProxyHeader = parseString(value, defaultAuthProxyHeader)
case "AUTH_PROXY_USER_CREATION":

2
database/migration.go
View File

@ -12,7 +12,7 @@ import (
"miniflux.app/logger"
)
const schemaVersion = 34
const schemaVersion = 35
// Migrate executes database migrations.
func Migrate(db *sql.DB) {

3
database/sql.go
View File

@ -187,6 +187,8 @@ create index entries_user_feed_idx on entries (user_id, feed_id);
`,
"schema_version_33": `alter table users add column show_reading_time boolean default 't';`,
"schema_version_34": `CREATE INDEX entries_id_user_status_idx ON entries USING btree (id, user_id, status);`,
"schema_version_35": `alter table feeds add column fetch_via_proxy bool default false;
`,
"schema_version_4": `create type entry_sorting_direction as enum('asc', 'desc');
alter table users add column entry_direction entry_sorting_direction default 'asc';
`,
@ -244,6 +246,7 @@ var SqlMapChecksums = map[string]string{
"schema_version_32": "5b4de8dd2d7e3c6ae4150e0e3931df2ee989f2c667145bd67294e5a5f3fae456",
"schema_version_33": "bf38514efeb6c12511f41b1cc484f92722240b0a6ae874c32a958dfea3433d02",
"schema_version_34": "1a3e036f652fc98b7564a27013f04e1eb36dd0d68893c723168f134dc1065822",
"schema_version_35": "162a55df78eed4b9c9c141878132d5f1d97944b96f35a79e38f55716cdd6b3d2",
"schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
"schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",
"schema_version_6": "9d05b4fb223f0e60efc716add5048b0ca9c37511cf2041721e20505d6d798ce4",

1
database/sql/schema_version_35.sql Normal file
View File

@ -0,0 +1 @@
alter table feeds add column fetch_via_proxy bool default false;

22
http/client/client.go
View File

@ -47,6 +47,7 @@ type Client struct {
password string
userAgent string
Insecure bool
fetchViaProxy bool
}
func (c *Client) String() string {
@ -93,6 +94,12 @@ func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client
return c
}
// WithProxy enable proxy for current HTTP client request.
func (c *Client) WithProxy() *Client {
c.fetchViaProxy = true
return c
}
// WithUserAgent defines the User-Agent header to use for outgoing requests.
func (c *Client) WithUserAgent(userAgent string) *Client {
if userAgent != "" {
@ -230,12 +237,23 @@ func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, err
func (c *Client) buildClient() http.Client {
client := http.Client{Timeout: time.Duration(config.Opts.HTTPClientTimeout()) * time.Second}
transport := &http.Transport{}
if c.Insecure {
client.Transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
if c.fetchViaProxy && config.Opts.HasHTTPClientProxyConfigured() {
proxyURL, err := url.Parse(config.Opts.HTTPClientProxy())
if err != nil {
logger.Error("[HttpClient] Proxy URL error: %v", err)
} else {
logger.Debug("[HttpClient] Use proxy: %s", proxyURL)
transport.Proxy = http.ProxyURL(proxyURL)
}
}
client.Transport = transport
return client
}

33
locale/translations.go
View File

@ -254,6 +254,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
"form.category.label.title": "Titel",
"form.user.label.username": "Benutzername",
@ -599,6 +600,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",
"form.category.label.title": "Title",
"form.user.label.username": "Username",
@ -924,6 +926,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",
"form.category.label.title": "Título",
"form.user.label.username": "Nombre de usuario",
@ -1249,6 +1252,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
"form.category.label.title": "Titre",
"form.user.label.username": "Nom d'utilisateur",
@ -1594,6 +1598,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",
"form.category.label.title": "Titolo",
"form.user.label.username": "Nome utente",
@ -1919,6 +1924,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
"form.feed.label.disabled": "このフィードを更新しない",
"form.category.label.title": "タイトル",
"form.user.label.username": "ユーザー名",
@ -2244,6 +2250,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",
"form.category.label.title": "Naam",
"form.user.label.username": "Gebruikersnaam",
@ -2589,6 +2596,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
"form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Tytuł",
"form.user.label.username": "Nazwa użytkownika",
@ -2939,6 +2947,7 @@ var translations = map[string]string{
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
"form.category.label.title": "Título",
"form.user.label.username": "Nome de usuário",
"form.user.label.password": "Senha",
@ -3265,6 +3274,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокси",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Название",
"form.user.label.username": "Имя пользователя",
@ -3594,6 +3604,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "重写规则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
"form.feed.label.fetch_via_proxy": "通过代理获取",
"form.feed.label.disabled": "请勿刷新此Feed",
"form.category.label.title": "标题",
"form.user.label.username": "用户名",
@ -3684,15 +3695,15 @@ var translations = map[string]string{
}
var translationsChecksums = map[string]string{
"de_DE": "21e1bfb0f43d71efe38812b4337ddf6980c11ed18f4d06446ff7eda9dfa6b1f1",
"en_US": "30cbcb2170782f1e66f69066947bf053f68065d7b270eea879f2c573819dd52b",
"es_ES": "50dc7c8c2db7368bae133f5b455721470d314321153d41e4f27436a0f3f176e6",
"fr_FR": "373fd2db868961758bd1483c34f117b03aadea17080f268bc8bbd0acdfbc5eed",
"it_IT": "8d8f0bd75b4e7dec9370647c888dd9438b691130d9c41f839cdfff8cbc606cb5",
"ja_JP": "ec3a21c547e4625ad359624e43ba31b556fb8d8b8ff7fc7a20df089317db99b3",
"nl_NL": "20e180be2375f07ec02eb05f372a9102c13037a79e5651ce9bd41507fd2180d2",
"pl_PL": "b1526955641823708b4c1ca753b61e1e0561d0a3d33da3f62170540903031b0d",
"pt_BR": "cf8e131d39daac82d3157c6538c0643392a06358b7bc98be8579412ebd63f60e",
"ru_RU": "4056e4e94861835d44064273371adbbded7190e2b719769886eb99e6c9feaf82",
"zh_CN": "044abb0a34eee3d8d5597811d40166762311b8e4cd08b891796113790cc775f0",
"de_DE": "8f96cb46f5a7e8f64ee8f10176dc3a2f3d53953d250317da83a79d0700b47c82",
"en_US": "d33324caed406ecf6ce03920b15e235d46b258457a8bd48cd1ade685b9a3ad6b",
"es_ES": "2ff9333218dba2b86cb84f377dad66b9dc73848aee6bb09889cbdc10e58ca077",
"fr_FR": "07dc2cfdbc14cdf16312423158656f5526d3c3c7be490abf5503109a408e5056",
"it_IT": "d4f68a507e1deb9fab3aa38fb78d9e9e4040386d6f36611ec5f105adfb4b0d03",
"ja_JP": "5c4c063ebaee14bed941b020e0d19de5ef5e8d3bf11c1967b1f321d57d5af6a9",
"nl_NL": "f862027e192be7a09730470acc2639971c4abf01b256d5bb81246960cc54adcf",
"pl_PL": "35147e55f1800964d268dc04b9cc25a9c8fa98077f759c5d3a5bd339f0eee53e",
"pt_BR": "2461105ebc2a2d57b3a63a29ee21f74e3d1eba54c049abcfd077dd30acc8d0a2",
"ru_RU": "402f15d3c68e008a1ffa3dddb4002126a29c1cf93359a64bca944a8da541da72",
"zh_CN": "bbea61a08bec37518d8c8c7735c7b8001d011a1d43ddb15ab639fee11b45ca87",
}

1
locale/translations/de_DE.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
"form.category.label.title": "Titel",
"form.user.label.username": "Benutzername",

1
locale/translations/en_US.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",
"form.category.label.title": "Title",
"form.user.label.username": "Username",

1
locale/translations/es_ES.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",
"form.category.label.title": "Título",
"form.user.label.username": "Nombre de usuario",

1
locale/translations/fr_FR.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
"form.category.label.title": "Titre",
"form.user.label.username": "Nom d'utilisateur",

1
locale/translations/it_IT.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",
"form.category.label.title": "Titolo",
"form.user.label.username": "Nome utente",

1
locale/translations/ja_JP.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
"form.feed.label.disabled": "このフィードを更新しない",
"form.category.label.title": "タイトル",
"form.user.label.username": "ユーザー名",

1
locale/translations/nl_NL.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",
"form.category.label.title": "Naam",
"form.user.label.username": "Gebruikersnaam",

1
locale/translations/pl_PL.json
View File

@ -251,6 +251,7 @@
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
"form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Tytuł",
"form.user.label.username": "Nazwa użytkownika",

1
locale/translations/pt_BR.json
View File

@ -250,6 +250,7 @@
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
"form.category.label.title": "Título",
"form.user.label.username": "Nome de usuário",
"form.user.label.password": "Senha",

1
locale/translations/ru_RU.json
View File

@ -251,6 +251,7 @@
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокси",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Название",
"form.user.label.username": "Имя пользователя",

1
locale/translations/zh_CN.json
View File

@ -247,6 +247,7 @@
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "重写规则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
"form.feed.label.fetch_via_proxy": "通过代理获取",
"form.feed.label.disabled": "请勿刷新此Feed",
"form.category.label.title": "标题",
"form.user.label.username": "用户名",

5
miniflux.1
View File

@ -250,6 +250,11 @@ Maximum body size for HTTP requests in Mebibyte (MiB)\&.
.br
Default is 15 MiB\&.
.TP
.B HTTP_CLIENT_PROXY
Proxy URL for HTTP client\&.
.br
Default is empty\&.
.TP
.B AUTH_PROXY_HEADER
Proxy authentication HTTP header\&.
.TP

4
model/feed.go
View File

@ -34,6 +34,7 @@ type Feed struct {
Password string `json:"password"`
Disabled bool `json:"disabled"`
IgnoreHTTPCache bool `json:"ignore_http_cache"`
FetchViaProxy bool `json:"fetch_via_proxy"`
Category *Category `json:"category,omitempty"`
Entries Entries `json:"entries,omitempty"`
Icon *FeedIcon `json:"icon"`
@ -71,13 +72,14 @@ func (f *Feed) WithCategoryID(categoryID int64) {
}
// WithBrowsingParameters defines browsing parameters.
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string) {
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) {
f.Crawler = crawler
f.UserAgent = userAgent
f.Username = username
f.Password = password
f.ScraperRules = scraperRules
f.RewriteRules = rewriteRules
f.FetchViaProxy = fetchViaProxy
}
// WithError adds a new error message and increment the error counter.

2
model/feed_test.go
View File

@ -48,7 +48,7 @@ func TestFeedCategorySetter(t *testing.T) {
func TestFeedBrowsingParams(t *testing.T) {
feed := &Feed{}
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule")
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", false)
if !feed.Crawler {
t.Error(`The crawler must be activated`)

21
reader/feed/handler.go
View File

@ -34,7 +34,7 @@ type Handler struct {
}
// CreateFeed fetch, parse and store a new feed.
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string) (*model.Feed, error) {
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) (*model.Feed, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
if !h.store.CategoryExists(userID, categoryID) {
@ -44,6 +44,11 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
request := client.New(url)
request.WithCredentials(username, password)
request.WithUserAgent(userAgent)
if fetchViaProxy {
request.WithProxy()
}
response, requestErr := browser.Exec(request)
if requestErr != nil {
return nil, requestErr
@ -60,7 +65,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
subscription.UserID = userID
subscription.WithCategoryID(categoryID)
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules)
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, fetchViaProxy)
subscription.WithClientResponse(response)
subscription.CheckedNow()
@ -72,7 +77,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
logger.Debug("[Handler:CreateFeed] Feed saved with ID: %d", subscription.ID)
checkFeedIcon(h.store, subscription.ID, subscription.SiteURL)
checkFeedIcon(h.store, subscription.ID, subscription.SiteURL, fetchViaProxy)
return subscription, nil
}
@ -111,6 +116,10 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
request.WithCacheHeaders(originalFeed.EtagHeader, originalFeed.LastModifiedHeader)
}
if originalFeed.FetchViaProxy {
request.WithProxy()
}
response, requestErr := browser.Exec(request)
if requestErr != nil {
originalFeed.WithError(requestErr.Localize(printer))
@ -141,7 +150,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
// We update caching headers only if the feed has been modified,
// because some websites don't return the same headers when replying with a 304.
originalFeed.WithClientResponse(response)
checkFeedIcon(h.store, originalFeed.ID, originalFeed.SiteURL)
checkFeedIcon(h.store, originalFeed.ID, originalFeed.SiteURL, originalFeed.FetchViaProxy)
} else {
logger.Debug("[Handler:RefreshFeed] Feed #%d not modified", feedID)
}
@ -162,9 +171,9 @@ func NewFeedHandler(store *storage.Storage) *Handler {
return &Handler{store}
}
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string) {
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string, fetchViaProxy bool) {
if !store.HasIcon(feedID) {
icon, err := icon.FindIcon(websiteURL)
icon, err := icon.FindIcon(websiteURL, fetchViaProxy)
if err != nil {
logger.Debug("CheckFeedIcon: %v (feedID=%d websiteURL=%s)", err, feedID, websiteURL)
} else if icon == nil {

12
reader/icon/finder.go
View File

@ -21,9 +21,12 @@ import (
)
// FindIcon try to find the website's icon.
func FindIcon(websiteURL string) (*model.Icon, error) {
func FindIcon(websiteURL string, fetchViaProxy bool) (*model.Icon, error) {
rootURL := url.RootURL(websiteURL)
clt := client.New(rootURL)
if fetchViaProxy {
clt.WithProxy()
}
response, err := clt.Get()
if err != nil {
return nil, fmt.Errorf("unable to download website index page: %v", err)
@ -43,7 +46,7 @@ func FindIcon(websiteURL string) (*model.Icon, error) {
}
logger.Debug("[FindIcon] Fetching icon => %s", iconURL)
icon, err := downloadIcon(iconURL)
icon, err := downloadIcon(iconURL, fetchViaProxy)
if err != nil {
return nil, err
}
@ -86,8 +89,11 @@ func parseDocument(websiteURL string, data io.Reader) (string, error) {
return iconURL, nil
}
func downloadIcon(iconURL string) (*model.Icon, error) {
func downloadIcon(iconURL string, fetchViaProxy bool) (*model.Icon, error) {
clt := client.New(iconURL)
if fetchViaProxy {
clt.WithProxy()
}
response, err := clt.Get()
if err != nil {
return nil, fmt.Errorf("unable to download iconURL: %v", err)

7
reader/subscription/finder.go
View File

@ -26,13 +26,18 @@ var (
)
// FindSubscriptions downloads and try to find one or more subscriptions from an URL.
func FindSubscriptions(websiteURL, userAgent, username, password string) (Subscriptions, *errors.LocalizedError) {
func FindSubscriptions(websiteURL, userAgent, username, password string, fetchViaProxy bool) (Subscriptions, *errors.LocalizedError) {
websiteURL = findYoutubeChannelFeed(websiteURL)
websiteURL = parseYoutubeVideoPage(websiteURL)
request := client.New(websiteURL)
request.WithCredentials(username, password)
request.WithUserAgent(userAgent)
if fetchViaProxy {
request.WithProxy()
}
response, err := browser.Exec(request)
if err != nil {
return nil, err

17
storage/feed.go
View File

@ -32,6 +32,7 @@ var feedListQuery = `
f.username,
f.password,
f.ignore_http_cache,
f.fetch_via_proxy,
f.disabled,
f.category_id,
c.title as category_title,
@ -133,6 +134,7 @@ func (s *Storage) FeedsByCategoryWithCounters(userID, categoryID int64) (model.F
f.username,
f.password,
f.ignore_http_cache,
f.fetch_via_proxy,
f.disabled,
f.category_id,
c.title as category_title,
@ -242,6 +244,7 @@ func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{}
&feed.Username,
&feed.Password,
&feed.IgnoreHTTPCache,
&feed.FetchViaProxy,
&feed.Disabled,
&feed.Category.ID,
&feed.Category.Title,
@ -326,6 +329,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
f.username,
f.password,
f.ignore_http_cache,
f.fetch_via_proxy,
f.disabled,
f.category_id,
c.title as category_title,
@ -357,6 +361,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
&feed.Username,
&feed.Password,
&feed.IgnoreHTTPCache,
&feed.FetchViaProxy,
&feed.Disabled,
&feed.Category.ID,
&feed.Category.Title,
@ -396,10 +401,11 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
password,
disabled,
scraper_rules,
rewrite_rules
rewrite_rules,
fetch_via_proxy
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
RETURNING
id
`
@ -419,6 +425,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
feed.Disabled,
feed.ScraperRules,
feed.RewriteRules,
feed.FetchViaProxy,
).Scan(&feed.ID)
if err != nil {
return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)
@ -462,9 +469,10 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
password=$15,
disabled=$16,
next_check_at=$17,
ignore_http_cache=$18
ignore_http_cache=$18,
fetch_via_proxy=$19
WHERE
id=$19 AND user_id=$20
id=$20 AND user_id=$21
`
_, err = s.db.Exec(query,
feed.FeedURL,
@ -485,6 +493,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
feed.Disabled,
feed.NextCheckAt,
feed.IgnoreHTTPCache,
feed.FetchViaProxy,
feed.ID,
feed.UserID,
)

2
template/common.go
View File

@ -519,7 +519,7 @@ SOFTWARE.
var templateCommonMapChecksums = map[string]string{
"entry_pagination": "cdca9cf12586e41e5355190b06d9168f57f77b85924d1e63b13524bc15abcbf6",
"feed_list": "30acc9ecc413811e73a1dad120b5d44e29564de3ba794fb07ee886b30addfb19",
"feed_list": "931e43d328a116318c510de5658c688cd940b934c86b6ec82a472e1f81e020ae",
"feed_menu": "318d8662dda5ca9dfc75b909c8461e79c86fb5082df1428f67aaf856f19f4b50",
"icons": "3dbe754a98f524a227111191d76b8c6944711b13613cc548ee9e9808fe0bffb4",
"item_meta": "8306adf3ef9966de3e3dc74ca1042e51d778b027ab8cf0a60a2e94a0115982dc",

3
template/html/add_subscription.html
View File

@ -30,6 +30,9 @@
<summary>{{ t "page.add_feed.legend.advanced_options" }}</summary>
<div class="details-content">
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label for="form-user-agent">{{ t "form.feed.label.user_agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}" autocomplete="off">

3
template/html/choose_subscription.html
View File

@ -14,6 +14,9 @@
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
<input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
<input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
{{ if .form.FetchViaProxy }}
<input type="hidden" name="fetch_via_proxy" value="1">
{{ end }}
{{ if .form.Crawler }}
<input type="hidden" name="crawler" value="1">
{{ end }}

3
template/html/edit_feed.html
View File

@ -73,6 +73,9 @@
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
<label><input type="checkbox" name="ignore_http_cache" value="1" {{ if .form.IgnoreHTTPCache }}checked{{ end }}> {{ t "form.feed.label.ignore_http_cache" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label><input type="checkbox" name="disabled" value="1" {{ if .form.Disabled }}checked{{ end }}> {{ t "form.feed.label.disabled" }}</label>
<div class="buttons">

15
template/views.go
View File

@ -61,6 +61,9 @@ var templateViewsMap = map[string]string{
<summary>{{ t "page.add_feed.legend.advanced_options" }}</summary>
<div class="details-content">
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label for="form-user-agent">{{ t "form.feed.label.user_agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}" autocomplete="off">
@ -380,6 +383,9 @@ var templateViewsMap = map[string]string{
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
<input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
<input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
{{ if .form.FetchViaProxy }}
<input type="hidden" name="fetch_via_proxy" value="1">
{{ end }}
{{ if .form.Crawler }}
<input type="hidden" name="crawler" value="1">
{{ end }}
@ -592,6 +598,9 @@ var templateViewsMap = map[string]string{
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
<label><input type="checkbox" name="ignore_http_cache" value="1" {{ if .form.IgnoreHTTPCache }}checked{{ end }}> {{ t "form.feed.label.ignore_http_cache" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label><input type="checkbox" name="disabled" value="1" {{ if .form.Disabled }}checked{{ end }}> {{ t "form.feed.label.disabled" }}</label>
<div class="buttons">
@ -1548,18 +1557,18 @@ var templateViewsMap = map[string]string{
var templateViewsMapChecksums = map[string]string{
"about": "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc",
"add_subscription": "0dbea93b6fc07423fa066122ad960c69616b829533371a2dbadec1e22d4f1ae0",
"add_subscription": "63961a83964acca354bc30eaae1f5e80f410ae4091af8da317380d4298f79032",
"api_keys": "27d401b31a72881d5232486ba17eb47edaf5246eaedce81de88698c15ebb2284",
"bookmark_entries": "892fe6cbf5a3301416dfb76e62935b495ca194275cfe113105a85b40ce7c200f",
"categories": "9dfc3cb7bb91c7750753fe962ee4540dd1843e5f75f9e0a575ee964f6f9923e9",
"category_entries": "8fa0e0b8f85e2572c40dee855b6d636207c3561086b234c93100673774c06746",
"category_feeds": "07154127087f9b127f7290abad6020c35ad9ceb2490b869120b7628bc4413808",
"choose_subscription": "84c9730cadd78e6ee5a6b4c499aab33acddb4324ac01924d33387543eec4d702",
"choose_subscription": "22109d760ea8079c491561d0106f773c885efbf66f87d81fcf8700218260d2a0",
"create_api_key": "5f74d4e92a6684927f5305096378c8be278159a5cd88ce652c7be3280a7d1685",
"create_category": "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d",
"create_user": "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a",
"edit_category": "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36",
"edit_feed": "ff90b1883e2934e0236d530d8b778affe7665a6b08cdf9ae612c7e56469818ef",
"edit_feed": "7e86275f8e9325ddbffe79f6db871e58ad86d08c396e9b2ff8af69a09c4bf63b",
"edit_user": "c692db9de1a084c57b93e95a14b041d39bf489846cbb91fc982a62b72b77062a",
"entry": "c503dcf77de37090b9f05352bb9d99729085eec6e7bc22be94f2b4b244b4e48c",
"feed_entries": "ea5b88e3ad6b166d83b70e021d7b420d025f80decb6e24c79d13f8ce7c910b04",

3
ui/feed_edit.go
View File

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
@ -52,6 +53,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
Username: feed.Username,
Password: feed.Password,
IgnoreHTTPCache: feed.IgnoreHTTPCache,
FetchViaProxy: feed.FetchViaProxy,
Disabled: feed.Disabled,
}
@ -65,6 +67,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, view.Render("edit_feed"))
}

3
ui/form/feed.go
View File

@ -25,6 +25,7 @@ type FeedForm struct {
Username string
Password string
IgnoreHTTPCache bool
FetchViaProxy bool
Disabled bool
}
@ -51,6 +52,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
feed.Username = f.Username
feed.Password = f.Password
feed.IgnoreHTTPCache = f.IgnoreHTTPCache
feed.FetchViaProxy = f.FetchViaProxy
feed.Disabled = f.Disabled
return feed
}
@ -74,6 +76,7 @@ func NewFeedForm(r *http.Request) *FeedForm {
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
IgnoreHTTPCache: r.FormValue("ignore_http_cache") == "1",
FetchViaProxy: r.FormValue("fetch_via_proxy") == "1",
Disabled: r.FormValue("disabled") == "1",
}
}

2
ui/form/subscription.go
View File

@ -16,6 +16,7 @@ type SubscriptionForm struct {
URL string
CategoryID int64
Crawler bool
FetchViaProxy bool
UserAgent string
Username string
Password string
@ -42,6 +43,7 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
return &SubscriptionForm{
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
FetchViaProxy: r.FormValue("fetch_via_proxy") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),

4
ui/subscription_add.go
View File

@ -7,9 +7,10 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/response/html"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
"miniflux.app/ui/form"
"miniflux.app/ui/session"
"miniflux.app/ui/view"
@ -38,6 +39,7 @@ func (h *handler) showAddSubscriptionPage(w http.ResponseWriter, r *http.Request
view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
view.Set("form", &form.SubscriptionForm{CategoryID: 0})
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, view.Render("add_subscription"))
}

2
ui/subscription_bookmarklet.go
View File

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
@ -40,6 +41,7 @@ func (h *handler) bookmarklet(w http.ResponseWriter, r *http.Request) {
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, view.Render("add_subscription"))
}

1
ui/subscription_choose.go
View File

@ -57,6 +57,7 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {
view.Set("form", subscriptionForm)

5
ui/subscription_submit.go
View File

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
@ -40,6 +41,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
v.Set("countUnread", h.store.CountUnreadEntries(user.ID))
v.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
v.Set("defaultUserAgent", client.DefaultUserAgent)
v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
subscriptionForm := form.NewSubscriptionForm(r)
if err := subscriptionForm.Validate(); err != nil {
@ -54,6 +56,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
subscriptionForm.UserAgent,
subscriptionForm.Username,
subscriptionForm.Password,
subscriptionForm.FetchViaProxy,
)
if findErr != nil {
logger.Error("[UI:SubmitSubscription] %s", findErr)
@ -82,6 +85,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {
v.Set("form", subscriptionForm)
@ -99,6 +103,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
v.Set("user", user)
v.Set("countUnread", h.store.CountUnreadEntries(user.ID))
v.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, v.Render("choose_subscription"))
}