mirrors
/
miniflux-v2
mirror of https://github.com/miniflux/v2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add feed filters (Keeplist and Blocklist)

This commit is contained in:
Manuel Garrido 2020-10-16 22:40:56 +01:00 committed by GitHub
parent 3afdf25012
commit 84b83fc3c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 359 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/feed.go
View File

@ -51,6 +51,8 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) {
feedInfo.Password,
feedInfo.ScraperRules,
feedInfo.RewriteRules,
feedInfo.BlocklistRules,
feedInfo.KeeplistRules,
feedInfo.FetchViaProxy,
)
if err != nil {

52
api/payload.go
View File

@ -24,15 +24,17 @@ type entriesResponse struct {
}
type feedCreation struct {
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
FetchViaProxy bool `json:"fetch_via_proxy"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
FetchViaProxy bool `json:"fetch_via_proxy"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
BlocklistRules string `json:"blocklist_rules"`
KeeplistRules string `json:"keeplist_rules"`
}
type subscriptionDiscovery struct {
@ -44,17 +46,19 @@ type subscriptionDiscovery struct {
}
type feedModification struct {
FeedURL *string `json:"feed_url"`
SiteURL *string `json:"site_url"`
Title *string `json:"title"`
ScraperRules *string `json:"scraper_rules"`
RewriteRules *string `json:"rewrite_rules"`
Crawler *bool `json:"crawler"`
UserAgent *string `json:"user_agent"`
Username *string `json:"username"`
Password *string `json:"password"`
CategoryID *int64 `json:"category_id"`
Disabled *bool `json:"disabled"`
FeedURL *string `json:"feed_url"`
SiteURL *string `json:"site_url"`
Title *string `json:"title"`
ScraperRules *string `json:"scraper_rules"`
RewriteRules *string `json:"rewrite_rules"`
BlocklistRules *string `json:"blocklist_rules"`
KeeplistRules *string `json:"keeplist_rules"`
Crawler *bool `json:"crawler"`
UserAgent *string `json:"user_agent"`
Username *string `json:"username"`
Password *string `json:"password"`
CategoryID *int64 `json:"category_id"`
Disabled *bool `json:"disabled"`
}
func (f *feedModification) Update(feed *model.Feed) {
@ -78,6 +82,14 @@ func (f *feedModification) Update(feed *model.Feed) {
feed.RewriteRules = *f.RewriteRules
}
if f.KeeplistRules != nil {
feed.KeeplistRules = *f.KeeplistRules
}
if f.BlocklistRules != nil {
feed.BlocklistRules = *f.BlocklistRules
}
if f.Crawler != nil {
feed.Crawler = *f.Crawler
}

24
client/core.go
View File

@ -92,6 +92,8 @@ type Feed struct {
ParsingErrorCount int `json:"parsing_error_count,omitempty"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
BlocklistRules string `json:"blocklist_rules"`
KeeplistRules string `json:"keeplist_rules"`
Crawler bool `json:"crawler"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
@ -101,16 +103,18 @@ type Feed struct {
// FeedModification represents changes for a feed.
type FeedModification struct {
FeedURL *string `json:"feed_url"`
SiteURL *string `json:"site_url"`
Title *string `json:"title"`
ScraperRules *string `json:"scraper_rules"`
RewriteRules *string `json:"rewrite_rules"`
Crawler *bool `json:"crawler"`
UserAgent *string `json:"user_agent"`
Username *string `json:"username"`
Password *string `json:"password"`
CategoryID *int64 `json:"category_id"`
FeedURL *string `json:"feed_url"`
SiteURL *string `json:"site_url"`
Title *string `json:"title"`
ScraperRules *string `json:"scraper_rules"`
RewriteRules *string `json:"rewrite_rules"`
BlocklistRules *string `json:"blocklist_rules"`
KeeplistRules *string `json:"keeplist_rules"`
Crawler *bool `json:"crawler"`
UserAgent *string `json:"user_agent"`
Username *string `json:"username"`
Password *string `json:"password"`
CategoryID *int64 `json:"category_id"`
}
// FeedIcon represents the feed icon.

2
database/migration.go
View File

@ -12,7 +12,7 @@ import (
"miniflux.app/logger"
)
const schemaVersion = 39
const schemaVersion = 40
// Migrate executes database migrations.
func Migrate(db *sql.DB) {

6
database/sql.go
View File

@ -197,6 +197,11 @@ create index entries_user_feed_idx on entries (user_id, feed_id);
`,
"schema_version_4": `create type entry_sorting_direction as enum('asc', 'desc');
alter table users add column entry_direction entry_sorting_direction default 'asc';
`,
"schema_version_40": `alter table feeds
add column blocklist_rules text not null default '',
add column keeplist_rules text not null default ''
;
`,
"schema_version_5": `create table integrations (
user_id int not null,
@ -258,6 +263,7 @@ var SqlMapChecksums = map[string]string{
"schema_version_38": "e91d2f4075ceb7b8a16a25f350f36dee12cfd1ad86b8b6414c4cf2e9a003358c",
"schema_version_39": "b0f90b97502921d4681a07c64d180a91a0b4ccac7d3c1dbe30519ad6f1bf1737",
"schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
"schema_version_40": "6a8fec92399f853ed6817aff4cfa43255dce4c19afad796e41519d09de62105e",
"schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",
"schema_version_6": "9d05b4fb223f0e60efc716add5048b0ca9c37511cf2041721e20505d6d798ce4",
"schema_version_7": "33f298c9aa30d6de3ca28e1270df51c2884d7596f1283a75716e2aeb634cd05c",

4
database/sql/schema_version_40.sql Normal file
View File

@ -0,0 +1,4 @@
alter table feeds
add column blocklist_rules text not null default '',
add column keeplist_rules text not null default ''
;

11
go.sum
View File

@ -348,7 +348,6 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BG
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -362,7 +361,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -371,14 +369,12 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c h1:38q6VNPWR010vN82/SB121GujZNIfAUb4YttE2rhGuc=
golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@ -399,14 +395,11 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@ -428,16 +421,13 @@ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLY
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
@ -457,7 +447,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

45
locale/translations.go
View File

@ -253,6 +253,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Standardbenutzeragenten überschreiben",
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
"form.feed.label.blocklist_rules": "Regeln blockieren",
"form.feed.label.keeplist_rules": "Regeln einhalten",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
@ -600,6 +602,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Override Default User Agent",
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
"form.feed.label.blocklist_rules": "Block Rules",
"form.feed.label.keeplist_rules": "Keep Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",
@ -927,6 +931,9 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Invalidar el agente de usuario predeterminado",
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
"form.feed.label.blocklist_rules": "Reglas de Filtrado(Bloquear)",
"form.feed.label.keeplist_rules": "Reglas de Filtrado(Permitir)",
"form.feed.label.blocklist_rules": "Reglas de Blacklist",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",
@ -1254,6 +1261,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Remplacer l'agent utilisateur par défaut",
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
"form.feed.label.blocklist_rules": "Règles de blocage",
"form.feed.label.keeplist_rules": "Règles d'autorisation",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
@ -1601,6 +1610,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Usa user agent personalizzato",
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
"form.feed.label.blocklist_rules": "Regole di blocco",
"form.feed.label.keeplist_rules": "Regole di autorizzazione",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",
@ -1928,6 +1939,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "ディフォルトの User Agent を上書きする",
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
"form.feed.label.blocklist_rules": "ブロックルール",
"form.feed.label.keeplist_rules": "許可規則",
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
"form.feed.label.disabled": "このフィードを更新しない",
@ -2255,6 +2268,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Standaard User Agent overschrijven",
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
"form.feed.label.blocklist_rules": "Blokkeer regels",
"form.feed.label.keeplist_rules": "toestemmingsregels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",
@ -2602,6 +2617,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Zastąp domyślny agent użytkownika",
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
"form.feed.label.blocklist_rules": "Zasady blokowania",
"form.feed.label.keeplist_rules": "Zasady zezwoleń",
"form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Не обновлять этот канал",
@ -2953,6 +2970,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Sobrescrever o agente de usuário (user-agent) padrão",
"form.feed.label.scraper_rules": "Regras do scraper",
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
"form.feed.label.blocklist_rules": "Regras de bloqueio",
"form.feed.label.keeplist_rules": "Regras de permissão",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
@ -3282,6 +3301,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Переопределить User Agent по умолчанию",
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
"form.feed.label.blocklist_rules": "Правила блокировки",
"form.feed.label.keeplist_rules": "правила разрешений",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокси",
"form.feed.label.disabled": "Не обновлять этот канал",
@ -3613,6 +3634,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "覆盖默认 User-Agent",
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "重写规则",
"form.feed.label.blocklist_rules": "封锁规则",
"form.feed.label.keeplist_rules": "许可规则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
"form.feed.label.fetch_via_proxy": "通过代理获取",
"form.feed.label.disabled": "请勿刷新此Feed",
@ -3706,15 +3729,15 @@ var translations = map[string]string{
}
var translationsChecksums = map[string]string{
"de_DE": "dbd9141c3b2f436d16ff660735904897c235406094f0e7d70cbceb509c6063b6",
"en_US": "61ef12ca29c271905a594ba1fe600871c930e7cfa25f0721bf076cd056cf60d4",
"es_ES": "74e1b08adc78c9bd2fe05df69d19232fca86cb5fc94379ee42cef34878e4b8e5",
"fr_FR": "3b61ad39a8d5227aac7b49897a6f78828c7e4a2040fe9636af932515c89d9785",
"it_IT": "048498833a7c9bb519cde0a61f955e15cbcf117b2006955305fb23cc652ed9f4",
"ja_JP": "cd8cac53f606066d60b3bc5586c0a484ee97466e3cca4816228ce74faf5916fe",
"nl_NL": "b76c996efdbc814688ad3db50a906397d45717b19f49a4702b4e65f50df23600",
"pl_PL": "a27aecbb6cd96d0714b2c934f5d96e01ab229e031e9e5ea0de96cec8c5cae0bf",
"pt_BR": "8e3cdaa0e39485a6eb5cf3942ec23b8f6a16e8efc551f52a35f74e444ebd70e3",
"ru_RU": "92d224b6563777efe64e1972884547f27e7374074b9f4f1fc2eeb331f6c5597c",
"zh_CN": "de59177700df00c27352d785c62f66ee49955b8b294f0eaf944ad7c4025838c7",
"de_DE": "8acdf65175293ab3684f945582f9f4adb442b5008afba3312a100bb75c7910d8",
"en_US": "0fdf8969fa460ffb6c21ad8afd624eef130dcd1bb18c489e3a7fd38e6ed2d563",
"es_ES": "86e8bf5fab817c536aa73bdf2aa0f5c09ddad6f7457b662a5e091304815aac05",
"fr_FR": "020deb5822f8bec9d94c9b9a387651efaef21f21239e7c203d76c71260bbbd54",
"it_IT": "77c5963a80c173e2257909b47e46cf7f1ecbbae773c61454fcc987d8b26e4a2a",
"ja_JP": "f2b6b61ac847f17031ab34284a3f3b6355c03fd1534164024e33d1659126e227",
"nl_NL": "4dba877744bb92911fc488c3cfa47aad605501d1ba351f7136b92cb4872fbd34",
"pl_PL": "10acf265cea096636f640d0e275b136fd06cf992d8a306eb418d6dcd21cb8b24",
"pt_BR": "dc853281eb52ae1225a36a9bd2c6bb3c62304834b3b1c092edcdec86823468b4",
"ru_RU": "46d039f6d50e48d841d09285d28810c609cd62dd821da4c68798490d57d1d090",
"zh_CN": "80e8a170246b0b90dffda393d897760ecb4e94b264cacc124db2be6845ac9361",
}

2
locale/translations/de_DE.json
View File

@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Standardbenutzeragenten überschreiben",
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
"form.feed.label.blocklist_rules": "Regeln blockieren",
"form.feed.label.keeplist_rules": "Regeln einhalten",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",

2
locale/translations/en_US.json
View File

@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Override Default User Agent",
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
"form.feed.label.blocklist_rules": "Block Rules",
"form.feed.label.keeplist_rules": "Keep Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",

3
locale/translations/es_ES.json
View File

@ -248,6 +248,9 @@
"form.feed.label.user_agent": "Invalidar el agente de usuario predeterminado",
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
"form.feed.label.blocklist_rules": "Reglas de Filtrado(Bloquear)",
"form.feed.label.keeplist_rules": "Reglas de Filtrado(Permitir)",
"form.feed.label.blocklist_rules": "Reglas de Blacklist",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",

2
locale/translations/fr_FR.json
View File

@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Remplacer l'agent utilisateur par défaut",
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
"form.feed.label.blocklist_rules": "Règles de blocage",
"form.feed.label.keeplist_rules": "Règles d'autorisation",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",

2
locale/translations/it_IT.json
View File

@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Usa user agent personalizzato",
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
"form.feed.label.blocklist_rules": "Regole di blocco",
"form.feed.label.keeplist_rules": "Regole di autorizzazione",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",

2
locale/translations/ja_JP.json
View File

@ -248,6 +248,8 @@
"form.feed.label.user_agent": "ディフォルトの User Agent を上書きする",
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
"form.feed.label.blocklist_rules": "ブロックルール",
"form.feed.label.keeplist_rules": "許可規則",
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
"form.feed.label.disabled": "このフィードを更新しない",

2
locale/translations/nl_NL.json
View File

@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Standaard User Agent overschrijven",
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
"form.feed.label.blocklist_rules": "Blokkeer regels",
"form.feed.label.keeplist_rules": "toestemmingsregels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",

2
locale/translations/pl_PL.json
View File

@ -250,6 +250,8 @@
"form.feed.label.user_agent": "Zastąp domyślny agent użytkownika",
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
"form.feed.label.blocklist_rules": "Zasady blokowania",
"form.feed.label.keeplist_rules": "Zasady zezwoleń",
"form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Не обновлять этот канал",

2
locale/translations/pt_BR.json
View File

@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Sobrescrever o agente de usuário (user-agent) padrão",
"form.feed.label.scraper_rules": "Regras do scraper",
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
"form.feed.label.blocklist_rules": "Regras de bloqueio",
"form.feed.label.keeplist_rules": "Regras de permissão",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",

2
locale/translations/ru_RU.json
View File

@ -250,6 +250,8 @@
"form.feed.label.user_agent": "Переопределить User Agent по умолчанию",
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
"form.feed.label.blocklist_rules": "Правила блокировки",
"form.feed.label.keeplist_rules": "правила разрешений",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокси",
"form.feed.label.disabled": "Не обновлять этот канал",

2
locale/translations/zh_CN.json
View File

@ -246,6 +246,8 @@
"form.feed.label.user_agent": "覆盖默认 User-Agent",
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "重写规则",
"form.feed.label.blocklist_rules": "封锁规则",
"form.feed.label.keeplist_rules": "许可规则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
"form.feed.label.fetch_via_proxy": "通过代理获取",
"form.feed.label.disabled": "请勿刷新此Feed",

6
model/feed.go
View File

@ -29,6 +29,8 @@ type Feed struct {
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
Crawler bool `json:"crawler"`
BlocklistRules string `json:"blocklist_rules"`
KeeplistRules string `json:"keeplist_rules"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
@ -72,7 +74,7 @@ func (f *Feed) WithCategoryID(categoryID int64) {
}
// WithBrowsingParameters defines browsing parameters.
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) {
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules, blacklistRules, keeplistRules string, fetchViaProxy bool) {
f.Crawler = crawler
f.UserAgent = userAgent
f.Username = username
@ -80,6 +82,8 @@ func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, passwor
f.ScraperRules = scraperRules
f.RewriteRules = rewriteRules
f.FetchViaProxy = fetchViaProxy
f.BlocklistRules = blacklistRules
f.KeeplistRules = keeplistRules
}
// WithError adds a new error message and increment the error counter.

2
model/feed_test.go
View File

@ -48,7 +48,7 @@ func TestFeedCategorySetter(t *testing.T) {
func TestFeedBrowsingParams(t *testing.T) {
feed := &Feed{}
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", false)
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", "Look a Rule", "Oh wow another Rule", false)
if !feed.Crawler {
t.Error(`The crawler must be activated`)

4
reader/feed/handler.go
View File

@ -34,7 +34,7 @@ type Handler struct {
}
// CreateFeed fetch, parse and store a new feed.
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) (*model.Feed, error) {
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules, blocklistRules, keeplistRules string, fetchViaProxy bool) (*model.Feed, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
if !h.store.CategoryExists(userID, categoryID) {
@ -65,7 +65,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
subscription.UserID = userID
subscription.WithCategoryID(categoryID)
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, fetchViaProxy)
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, blocklistRules, keeplistRules, fetchViaProxy)
subscription.WithClientResponse(response)
subscription.CheckedNow()

36
reader/processor/processor.go
View File

@ -5,6 +5,7 @@
package processor
import (
"regexp"
"time"
"miniflux.app/config"
@ -19,9 +20,11 @@ import (
// ProcessFeedEntries downloads original web page for entries and apply filters.
func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
filterFeedEntries(feed)
for _, entry := range feed.Entries {
logger.Debug("[Feed #%d] Processing entry %s", feed.ID, entry.URL)
if feed.Crawler {
if !store.EntryURLExists(feed.ID, entry.URL) {
startTime := time.Now()
@ -51,6 +54,37 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
}
}
/*
Filters feed entries based on regex rules
First we filter based on our keep list, then we remove those entries that match the block list
*/
func filterFeedEntries(feed *model.Feed) {
var filteredEntries []*model.Entry
if len(feed.KeeplistRules) > 0 {
for _, entry := range feed.Entries {
match, _ := regexp.MatchString(feed.KeeplistRules, entry.Title)
if match == true {
filteredEntries = append(filteredEntries, entry)
}
}
} else {
filteredEntries = feed.Entries
}
if len(feed.BlocklistRules) > 0 {
k := 0
for _, entry := range filteredEntries {
match, _ := regexp.MatchString(feed.BlocklistRules, entry.Title)
if match != true {
filteredEntries[k] = entry
k++
}
}
filteredEntries = filteredEntries[:k]
}
feed.Entries = filteredEntries
}
// ProcessEntryWebPage downloads the entry web page and apply rewrite rules.
func ProcessEntryWebPage(entry *model.Entry) error {
startTime := time.Now()

88
reader/processor/processor_test.go Normal file
View File

@ -0,0 +1,88 @@
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package processor // import "miniflux.app/reader/processor"
import (
"miniflux.app/reader/parser"
"testing"
)
func TestKeeplistRules(t *testing.T) {
data := `<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>SomeGood News</title>
<link>http://foo.bar/</link>
<item>
<title>Kitten News</title>
<link>http://kitties.today/daily-kitten</link>
<description>Kitten picture of the day.</description>
<pubDate>Tue, 03 Jun 2003 09:39:21 GMT</pubDate>
<guid>http://kitties.today</guid>
</item>
<item>
<title>Daily Covid DoomScrolling News</title>
<link>http://covid.doom/daily-panic-dose</link>
<description>Did you know that you can get COVID IN YOUR DREAMS?.</description>
<pubDate>Tue, 03 Jun 2020 09:39:21 GMT</pubDate>
<guid>http://covid.doom</guid>
</item>
</channel>
</rss>`
feed, err := parser.ParseFeed(data)
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 2 {
t.Errorf("Error parsing feed")
}
//case insensitive
feed.KeeplistRules = "(?i)kitten"
filterFeedEntries(feed)
if len(feed.Entries) != 1 {
t.Errorf("Keeplist filter rule did not properly filter the feed")
}
}
func TestBlocklistRules(t *testing.T) {
data := `<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>SomeGood News</title>
<link>http://foo.bar/</link>
<item>
<title>Kitten News</title>
<link>http://kitties.today/daily-kitten</link>
<description>Kitten picture of the day.</description>
<pubDate>Tue, 03 Jun 2003 09:39:21 GMT</pubDate>
<guid>http://kitties.today</guid>
</item>
<item>
<title>Daily Covid DoomScrolling News</title>
<link>http://covid.doom/daily-panic-dose</link>
<description>Did you know that you can get COVID IN YOUR DREAMS?.</description>
<pubDate>Tue, 03 Jun 2020 09:39:21 GMT</pubDate>
<guid>http://covid.doom</guid>
</item>
</channel>
</rss>`
feed, err := parser.ParseFeed(data)
if err != nil {
t.Error(err)
}
if len(feed.Entries) != 2 {
t.Errorf("Error parsing feed")
}
//case insensitive
feed.BlocklistRules = "(?i)covid"
filterFeedEntries(feed)
if len(feed.Entries) != 1 {
t.Errorf("Keeplist filter rule did not properly filter the feed")
}
}

38
storage/feed.go
View File

@ -27,6 +27,8 @@ var feedListQuery = `
f.parsing_error_msg,
f.scraper_rules,
f.rewrite_rules,
f.blocklist_rules,
f.keeplist_rules,
f.crawler,
f.user_agent,
f.username,
@ -180,6 +182,8 @@ func (s *Storage) FeedsByCategoryWithCounters(userID, categoryID int64) (model.F
f.parsing_error_msg,
f.scraper_rules,
f.rewrite_rules,
f.blocklist_rules,
f.keeplist_rules,
f.crawler,
f.user_agent,
f.username,
@ -290,6 +294,8 @@ func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{}
&feed.ParsingErrorMsg,
&feed.ScraperRules,
&feed.RewriteRules,
&feed.BlocklistRules,
&feed.KeeplistRules,
&feed.Crawler,
&feed.UserAgent,
&feed.Username,
@ -375,6 +381,8 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
f.parsing_error_msg,
f.scraper_rules,
f.rewrite_rules,
f.blocklist_rules,
f.keeplist_rules,
f.crawler,
f.user_agent,
f.username,
@ -407,6 +415,8 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
&feed.ParsingErrorMsg,
&feed.ScraperRules,
&feed.RewriteRules,
&feed.BlocklistRules,
&feed.KeeplistRules,
&feed.Crawler,
&feed.UserAgent,
&feed.Username,
@ -453,10 +463,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
disabled,
scraper_rules,
rewrite_rules,
blocklist_rules,
keeplist_rules,
fetch_via_proxy
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
RETURNING
id
`
@ -476,6 +488,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
feed.Disabled,
feed.ScraperRules,
feed.RewriteRules,
feed.BlocklistRules,
feed.KeeplistRules,
feed.FetchViaProxy,
).Scan(&feed.ID)
if err != nil {
@ -523,16 +537,18 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
parsing_error_count=$9,
scraper_rules=$10,
rewrite_rules=$11,
crawler=$12,
user_agent=$13,
username=$14,
password=$15,
disabled=$16,
next_check_at=$17,
ignore_http_cache=$18,
fetch_via_proxy=$19
blocklist_rules=$12,
keeplist_rules=$13,
crawler=$14,
user_agent=$15,
username=$16,
password=$17,
disabled=$18,
next_check_at=$19,
ignore_http_cache=$20,
fetch_via_proxy=$21
WHERE
id=$20 AND user_id=$21
id=$22 AND user_id=$23
`
_, err = s.db.Exec(query,
feed.FeedURL,
@ -546,6 +562,8 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
feed.ParsingErrorCount,
feed.ScraperRules,
feed.RewriteRules,
feed.BlocklistRules,
feed.KeeplistRules,
feed.Crawler,
feed.UserAgent,
feed.Username,

6
template/html/add_subscription.html
View File

@ -55,6 +55,12 @@
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
<label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
<input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
<label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
<input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
</div>
</details>

6
template/html/edit_feed.html
View File

@ -64,6 +64,12 @@
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
<label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
<input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
<label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
<input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
<label for="form-category">{{ t "form.feed.label.category" }}</label>
<select id="form-category" name="category_id">
{{ range .categories }}

16
template/views.go
View File

@ -86,6 +86,12 @@ var templateViewsMap = map[string]string{
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
<label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
<input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
<label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
<input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
</div>
</details>
@ -589,6 +595,12 @@ var templateViewsMap = map[string]string{
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
<label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
<input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
<label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
<input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
<label for="form-category">{{ t "form.feed.label.category" }}</label>
<select id="form-category" name="category_id">
{{ range .categories }}
@ -1582,7 +1594,7 @@ var templateViewsMap = map[string]string{
var templateViewsMapChecksums = map[string]string{
"about": "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc",
"add_subscription": "63961a83964acca354bc30eaae1f5e80f410ae4091af8da317380d4298f79032",
"add_subscription": "82bf0dfadd64d3b6eda323a8174dd1446665b0804d27ca8718a828488627b287",
"api_keys": "27d401b31a72881d5232486ba17eb47edaf5246eaedce81de88698c15ebb2284",
"bookmark_entries": "eacbbdce7fa85ec66c4c12f02879daab562a17ff79f1aac1805617e83e3a3a42",
"categories": "9dfc3cb7bb91c7750753fe962ee4540dd1843e5f75f9e0a575ee964f6f9923e9",
@ -1593,7 +1605,7 @@ var templateViewsMapChecksums = map[string]string{
"create_category": "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d",
"create_user": "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a",
"edit_category": "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36",
"edit_feed": "7e86275f8e9325ddbffe79f6db871e58ad86d08c396e9b2ff8af69a09c4bf63b",
"edit_feed": "5de7626448c48de384a0388227ab0c3b75b1ec19b5de440c91039180852cc5dc",
"edit_user": "c692db9de1a084c57b93e95a14b041d39bf489846cbb91fc982a62b72b77062a",
"entry": "c503dcf77de37090b9f05352bb9d99729085eec6e7bc22be94f2b4b244b4e48c",
"feed_entries": "89977ea86b8d43305d587b70e6d9c45c2c88249b3966f2d31051dc7a5f1c48b6",

25
tests/feed_test.go
View File

@ -195,6 +195,31 @@ func TestUpdateFeedRewriteRules(t *testing.T) {
}
}
func TestUpdateFeedKeeplistRules(t *testing.T) {
client := createClient(t)
feed, _ := createFeed(t, client)
keeplistRules := "test"
updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModification{KeeplistRules: &keeplistRules})
if err != nil {
t.Fatal(err)
}
if updatedFeed.KeeplistRules != keeplistRules {
t.Fatalf(`Wrong KeeplistRules value, got "%v" instead of "%v"`, updatedFeed.KeeplistRules, keeplistRules)
}
keeplistRules = ""
updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModification{KeeplistRules: &keeplistRules})
if err != nil {
t.Fatal(err)
}
if updatedFeed.KeeplistRules != keeplistRules {
t.Fatalf(`Wrong KeeplistRules value, got "%v" instead of "%v"`, updatedFeed.KeeplistRules, keeplistRules)
}
}
func TestUpdateFeedUserAgent(t *testing.T) {
client := createClient(t)
feed, _ := createFeed(t, client)

2
ui/feed_edit.go
View File

@ -47,6 +47,8 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
Title: feed.Title,
ScraperRules: feed.ScraperRules,
RewriteRules: feed.RewriteRules,
BlocklistRules: feed.BlocklistRules,
KeeplistRules: feed.KeeplistRules,
Crawler: feed.Crawler,
UserAgent: feed.UserAgent,
CategoryID: feed.Category.ID,

7
ui/form/feed.go
View File

@ -19,6 +19,8 @@ type FeedForm struct {
Title string
ScraperRules string
RewriteRules string
BlocklistRules string
KeeplistRules string
Crawler bool
UserAgent string
CategoryID int64
@ -45,6 +47,8 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
feed.FeedURL = f.FeedURL
feed.ScraperRules = f.ScraperRules
feed.RewriteRules = f.RewriteRules
feed.BlocklistRules = f.BlocklistRules
feed.KeeplistRules = f.KeeplistRules
feed.Crawler = f.Crawler
feed.UserAgent = f.UserAgent
feed.ParsingErrorCount = 0
@ -63,7 +67,6 @@ func NewFeedForm(r *http.Request) *FeedForm {
if err != nil {
categoryID = 0
}
return &FeedForm{
FeedURL: r.FormValue("feed_url"),
SiteURL: r.FormValue("site_url"),
@ -71,6 +74,8 @@ func NewFeedForm(r *http.Request) *FeedForm {
ScraperRules: r.FormValue("scraper_rules"),
UserAgent: r.FormValue("user_agent"),
RewriteRules: r.FormValue("rewrite_rules"),
BlocklistRules: r.FormValue("blocklist_rules"),
KeeplistRules: r.FormValue("keeplist_rules"),
Crawler: r.FormValue("crawler") == "1",
CategoryID: int64(categoryID),
Username: r.FormValue("feed_username"),

39
ui/form/subscription.go
View File

@ -13,15 +13,17 @@ import (
// SubscriptionForm represents the subscription form.
type SubscriptionForm struct {
URL string
CategoryID int64
Crawler bool
FetchViaProxy bool
UserAgent string
Username string
Password string
ScraperRules string
RewriteRules string
URL string
CategoryID int64
Crawler bool
FetchViaProxy bool
UserAgent string
Username string
Password string
ScraperRules string
RewriteRules string
BlocklistRules string
KeeplistRules string
}
// Validate makes sure the form values are valid.
@ -41,14 +43,15 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
}
return &SubscriptionForm{
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
FetchViaProxy: r.FormValue("fetch_via_proxy") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
ScraperRules: r.FormValue("scraper_rules"),
RewriteRules: r.FormValue("rewrite_rules"),
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
ScraperRules: r.FormValue("scraper_rules"),
RewriteRules: r.FormValue("rewrite_rules"),
BlocklistRules: r.FormValue("blocklist_rules"),
KeeplistRules: r.FormValue("keeplist_rules"),
}
}

2
ui/subscription_choose.go
View File

@ -57,6 +57,8 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
subscriptionForm.BlocklistRules,
subscriptionForm.KeeplistRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {

2
ui/subscription_submit.go
View File

@ -85,6 +85,8 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
subscriptionForm.BlocklistRules,
subscriptionForm.KeeplistRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {