mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Implement Artist search with SQL, removed old search/indexer code

This commit is contained in:
Deluan 2020-01-13 16:02:49 -05:00 committed by Deluan Quintão
parent 3a9559a860
commit 614f4afe28
11 changed files with 45 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
api/wire_gen.go
View File

@ -7,9 +7,6 @@ package api
import (
"github.com/cloudsonic/sonic-server/itunesbridge"
"github.com/cloudsonic/sonic-server/persistence/db_ledis"
"github.com/deluan/gomate"
"github.com/deluan/gomate/ledis"
"github.com/google/wire"
)
@ -78,10 +75,5 @@ var allProviders = wire.NewSet(itunesbridge.NewItunesControl, NewSystemControlle
NewSearchingController,
NewUsersController,
NewMediaRetrievalController,
NewStreamController,
newDB, wire.FieldsOf(new(*Router), "Browser", "Cover", "ListGenerator", "Playlists", "Ratings", "Scrobbler", "Search"),
NewStreamController, wire.FieldsOf(new(*Router), "Browser", "Cover", "ListGenerator", "Playlists", "Ratings", "Scrobbler", "Search"),
)
func newDB() gomate.DB {
return ledis.NewEmbeddedDB(db_ledis.Db())
}

8
api/wire_injectors.go
View File

@ -4,9 +4,6 @@ package api
import (
"github.com/cloudsonic/sonic-server/itunesbridge"
"github.com/cloudsonic/sonic-server/persistence/db_ledis"
"github.com/deluan/gomate"
"github.com/deluan/gomate/ledis"
"github.com/google/wire"
)
@ -21,7 +18,6 @@ var allProviders = wire.NewSet(
NewUsersController,
NewMediaRetrievalController,
NewStreamController,
newDB,
wire.FieldsOf(new(*Router), "Browser", "Cover", "ListGenerator", "Playlists", "Ratings", "Scrobbler", "Search"),
)
@ -60,7 +56,3 @@ func initMediaRetrievalController(router *Router) *MediaRetrievalController {
func initStreamController(router *Router) *StreamController {
panic(wire.Build(allProviders))
}
func newDB() gomate.DB {
return ledis.NewEmbeddedDB(db_ledis.Db())
}

1
domain/artist.go
View File

@ -11,6 +11,7 @@ type ArtistRepository interface {
Put(m *Artist) error
Get(id string) (*Artist, error)
PurgeInactive(active Artists) ([]string, error)
Search(q string, offset int, size int) (Artists, error)
}
type Artists []Artist

100
engine/search.go
View File

@ -5,21 +5,10 @@ import (
"strings"
"github.com/cloudsonic/sonic-server/domain"
"github.com/cloudsonic/sonic-server/log"
"github.com/deluan/gomate"
"github.com/kennygrant/sanitize"
)
type Search interface {
ClearAll() error
IndexArtist(ar *domain.Artist) error
IndexAlbum(al *domain.Album) error
IndexMediaFile(mf *domain.MediaFile) error
RemoveArtist(ids ...string) error
RemoveAlbum(ids ...string) error
RemoveMediaFile(ids ...string) error
SearchArtist(ctx context.Context, q string, offset int, size int) (Entries, error)
SearchAlbum(ctx context.Context, q string, offset int, size int) (Entries, error)
SearchSong(ctx context.Context, q string, offset int, size int) (Entries, error)
@ -29,79 +18,22 @@ type search struct {
artistRepo domain.ArtistRepository
albumRepo domain.AlbumRepository
mfileRepo domain.MediaFileRepository
idxArtist gomate.Indexer
idxAlbum gomate.Indexer
idxSong gomate.Indexer
sArtist gomate.Searcher
sAlbum gomate.Searcher
sSong gomate.Searcher
}
func NewSearch(ar domain.ArtistRepository, alr domain.AlbumRepository, mr domain.MediaFileRepository, db gomate.DB) Search {
func NewSearch(ar domain.ArtistRepository, alr domain.AlbumRepository, mr domain.MediaFileRepository) Search {
s := &search{artistRepo: ar, albumRepo: alr, mfileRepo: mr}
s.idxArtist = gomate.NewIndexer(db, "gomate-artist-idx")
s.sArtist = gomate.NewSearcher(db, "gomate-artist-idx")
s.idxAlbum = gomate.NewIndexer(db, "gomate-album-idx")
s.sAlbum = gomate.NewSearcher(db, "gomate-album-idx")
s.idxSong = gomate.NewIndexer(db, "gomate-song-idx")
s.sSong = gomate.NewSearcher(db, "gomate-song-idx")
return s
}
func (s *search) ClearAll() error {
if err := s.idxArtist.Clear(); err != nil {
return err
}
if err := s.idxAlbum.Clear(); err != nil {
return err
}
if err := s.idxSong.Clear(); err != nil {
return err
}
return nil
}
func (s *search) IndexArtist(ar *domain.Artist) error {
return s.idxArtist.Index(ar.ID, sanitize.Accents(strings.ToLower(ar.Name)))
}
func (s *search) IndexAlbum(al *domain.Album) error {
return s.idxAlbum.Index(al.ID, sanitize.Accents(strings.ToLower(al.Name)))
}
func (s *search) IndexMediaFile(mf *domain.MediaFile) error {
return s.idxSong.Index(mf.ID, sanitize.Accents(strings.ToLower(mf.Title)))
}
func (s *search) RemoveArtist(ids ...string) error {
return s.idxArtist.Remove(ids...)
}
func (s *search) RemoveAlbum(ids ...string) error {
return s.idxAlbum.Remove(ids...)
}
func (s *search) RemoveMediaFile(ids ...string) error {
return s.idxSong.Remove(ids...)
}
func (s *search) SearchArtist(ctx context.Context, q string, offset int, size int) (Entries, error) {
q = sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*")))
min := offset
max := min + size - 1
resp, err := s.sArtist.Search(q, min, max)
resp, err := s.artistRepo.Search(q, offset, size)
if err != nil {
return nil, nil
}
res := make(Entries, 0, len(resp))
for _, id := range resp {
a, err := s.artistRepo.Get(id)
if criticalError(ctx, "Artist", id, err) {
return nil, err
}
if err == nil {
res = append(res, FromArtist(a))
}
for _, ar := range resp {
res = append(res, FromArtist(&ar))
}
return res, nil
}
@ -114,12 +46,7 @@ func (s *search) SearchAlbum(ctx context.Context, q string, offset int, size int
}
res := make(Entries, 0, len(resp))
for _, al := range resp {
if criticalError(ctx, "Album", al.ID, err) {
return nil, err
}
if err == nil {
res = append(res, FromAlbum(&al))
}
res = append(res, FromAlbum(&al))
}
return res, nil
}
@ -132,22 +59,7 @@ func (s *search) SearchSong(ctx context.Context, q string, offset int, size int)
}
res := make(Entries, 0, len(resp))
for _, mf := range resp {
if criticalError(ctx, "Song", mf.ID, err) {
return nil, err
}
if err == nil {
res = append(res, FromMediaFile(&mf))
}
res = append(res, FromMediaFile(&mf))
}
return res, nil
}
func criticalError(ctx context.Context, kind, id string, err error) bool {
switch {
case err != nil:
return true
case err == domain.ErrNotFound:
log.Warn(ctx, kind+"ID not in DB. Need a reindex?", "id", id)
}
return false
}

6
go.mod
View File

@ -4,13 +4,9 @@ go 1.13
require (
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/DataDog/zstd v1.4.4 // indirect
github.com/Masterminds/squirrel v1.1.0
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 // indirect
github.com/asdine/storm v2.1.2+incompatible
github.com/astaxie/beego v1.12.0
github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
github.com/deluan/gomate v0.0.0-20160327212459-3eb40643dd6f
github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131
github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a // indirect
github.com/dhowden/tag v0.0.0-20170128231422-9edd38ca5d10
@ -33,8 +29,6 @@ require (
github.com/smartystreets/goconvey v1.6.4
github.com/stretchr/testify v1.4.0 // indirect
github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c
github.com/vmihailenco/msgpack v4.0.1+incompatible // indirect
go.etcd.io/bbolt v1.3.3 // indirect
golang.org/x/sys v0.0.0-20200107162124-548cf772de50 // indirect
google.golang.org/appengine v1.6.5 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect

12
go.sum
View File

@ -1,15 +1,9 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE=
github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Masterminds/squirrel v1.1.0 h1:baP1qLdoQCeTw3ifCdOq2dkYc6vGcmRdaociKLbEJXs=
github.com/Masterminds/squirrel v1.1.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA=
github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM=
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM=
github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM=
github.com/asdine/storm v2.1.2+incompatible h1:dczuIkyqwY2LrtXPz8ixMrU/OFgZp71kbKTHGrXYt/Q=
github.com/asdine/storm v2.1.2+incompatible/go.mod h1:RarYDc9hq1UPLImuiXK3BIWPJLdIygvV3PsInK0FbVQ=
github.com/astaxie/beego v1.12.0 h1:MRhVoeeye5N+Flul5PoVfD9CslfdoH+xqC/xvSQ5u2Y=
github.com/astaxie/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o=
github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ=
@ -28,8 +22,6 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deluan/gomate v0.0.0-20160327212459-3eb40643dd6f h1:jZxJHFEzOavX4cM1BacQGZAMmhgHERXD7Qxyi2NLYtE=
github.com/deluan/gomate v0.0.0-20160327212459-3eb40643dd6f/go.mod h1:10VOt8RwQ8an9cSC2r77s1jqTucTHZSGN2wz46v+7ZM=
github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131 h1:siEGb+iB1Ea75U7BnkYVSqSRzE6QHlXCbqEXenxRmhQ=
github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131/go.mod h1:eVWQJVQ67aMvYhpkDwaH2Goy2vo6v8JCMfGXfQ9sPtw=
github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a h1:7MucP9rMAsQRcRE1sGpvMZoTxFYZlDmfDvCH+z7H+90=
@ -125,11 +117,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c h1:3eGShk3EQf5gJCYW+WzA0TEJQd37HLOmlYF7N0YJwv0=
github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
github.com/vmihailenco/msgpack v4.0.1+incompatible h1:RMF1enSPeKTlXrXdOcqjFUElywVZjjC6pqse21bKbEU=
github.com/vmihailenco/msgpack v4.0.1+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

3
persistence/db_ledis/artist_repository.go
View File

@ -34,5 +34,8 @@ func (r *artistRepository) PurgeInactive(active domain.Artists) ([]string, error
return e.(domain.Artist).ID
})
}
func (r *artistRepository) Search(q string, offset int, size int) (domain.Artists, error) {
return nil, errors.New("not implemented")
}
var _ domain.ArtistRepository = (*artistRepository)(nil)

28
persistence/db_sql/artist_repository.go
View File

@ -25,7 +25,11 @@ func NewArtistRepository() domain.ArtistRepository {
func (r *artistRepository) Put(a *domain.Artist) error {
ta := Artist(*a)
return WithTx(func(o orm.Ormer) error {
return r.put(o, a.ID, &ta)
err := r.put(o, a.ID, &ta)
if err != nil {
return err
}
return r.searcher.Index(o, r.tableName, a.ID, a.Name)
})
}
@ -48,5 +52,27 @@ func (r *artistRepository) PurgeInactive(activeList domain.Artists) ([]string, e
})
}
func (r *artistRepository) Search(q string, offset int, size int) (domain.Artists, error) {
if len(q) <= 2 {
return nil, nil
}
var results []Artist
err := r.searcher.Search(r.tableName, q, offset, size, &results, "name")
if err != nil {
return nil, err
}
return r.toArtists(results), nil
}
func (r *artistRepository) toArtists(all []Artist) domain.Artists {
result := make(domain.Artists, len(all))
for i, a := range all {
result[i] = domain.Artist(a)
}
return result
}
var _ domain.ArtistRepository = (*artistRepository)(nil)
var _ = domain.Artist(Artist{})

26
scanner/importer.go
View File

@ -9,7 +9,6 @@ import (
"github.com/cloudsonic/sonic-server/conf"
"github.com/cloudsonic/sonic-server/domain"
"github.com/cloudsonic/sonic-server/engine"
"github.com/cloudsonic/sonic-server/log"
"github.com/cloudsonic/sonic-server/utils"
)
@ -33,12 +32,11 @@ type Importer struct {
idxRepo domain.ArtistIndexRepository
plsRepo domain.PlaylistRepository
propertyRepo domain.PropertyRepository
search engine.Search
lastScan time.Time
lastCheck time.Time
}
func NewImporter(mediaFolder string, scanner Scanner, mfRepo domain.MediaFileRepository, albumRepo domain.AlbumRepository, artistRepo domain.ArtistRepository, idxRepo domain.ArtistIndexRepository, plsRepo domain.PlaylistRepository, propertyRepo domain.PropertyRepository, search engine.Search) *Importer {
func NewImporter(mediaFolder string, scanner Scanner, mfRepo domain.MediaFileRepository, albumRepo domain.AlbumRepository, artistRepo domain.ArtistRepository, idxRepo domain.ArtistIndexRepository, plsRepo domain.PlaylistRepository, propertyRepo domain.PropertyRepository) *Importer {
return &Importer{
scanner: scanner,
mediaFolder: mediaFolder,
@ -48,7 +46,6 @@ func NewImporter(mediaFolder string, scanner Scanner, mfRepo domain.MediaFileRep
idxRepo: idxRepo,
plsRepo: plsRepo,
propertyRepo: propertyRepo,
search: search,
}
}
@ -137,20 +134,14 @@ func (i *Importer) importLibrary() (err error) {
i.importArtistIndex()
log.Debug("Purging old data")
if deleted, err := i.mfRepo.PurgeInactive(mfs); err != nil {
if _, err := i.mfRepo.PurgeInactive(mfs); err != nil {
log.Error(err)
} else {
i.search.RemoveMediaFile(deleted...)
}
if deleted, err := i.albumRepo.PurgeInactive(als); err != nil {
if _, err := i.albumRepo.PurgeInactive(als); err != nil {
log.Error(err)
} else {
i.search.RemoveAlbum(deleted...)
}
if deleted, err := i.artistRepo.PurgeInactive(ars); err != nil {
if _, err := i.artistRepo.PurgeInactive(ars); err != nil {
log.Error("Deleting inactive artists", err)
} else {
i.search.RemoveArtist(deleted...)
}
if _, err := i.plsRepo.PurgeInactive(pls); err != nil {
log.Error(err)
@ -198,9 +189,6 @@ func (i *Importer) importMediaFiles() (domain.MediaFiles, int) {
if err := i.mfRepo.Put(mf); err != nil {
log.Error(err)
}
if err := i.search.IndexMediaFile(mf); err != nil {
log.Error("Error indexing artist", err)
}
updates++
if !i.lastScan.IsZero() {
log.Debug(fmt.Sprintf(`-- Updated Track: "%s"`, mf.Title))
@ -230,9 +218,6 @@ func (i *Importer) importAlbums() (domain.Albums, int) {
if err := i.albumRepo.Put(al); err != nil {
log.Error(err)
}
if err := i.search.IndexAlbum(al); err != nil {
log.Error("Error indexing artist", err)
}
updates++
if !i.lastScan.IsZero() {
log.Debug(fmt.Sprintf(`-- Updated Album: "%s" from "%s"`, al.Name, al.Artist))
@ -250,9 +235,6 @@ func (i *Importer) importArtists() domain.Artists {
if err := i.artistRepo.Put(ar); err != nil {
log.Error(err)
}
if err := i.search.IndexArtist(ar); err != nil {
log.Error("Error indexing artist", err)
}
}
return ars
}

15
wire_gen.go
View File

@ -14,8 +14,6 @@ import (
"github.com/cloudsonic/sonic-server/persistence/db_ledis"
"github.com/cloudsonic/sonic-server/persistence/db_sql"
"github.com/cloudsonic/sonic-server/scanner"
"github.com/deluan/gomate"
"github.com/deluan/gomate/ledis"
"github.com/google/wire"
)
@ -31,9 +29,7 @@ func CreateApp(musicFolder string, p persistence.ProviderIdentifier) *App {
artistIndexRepository := provider.ArtistIndexRepository
playlistRepository := provider.PlaylistRepository
propertyRepository := provider.PropertyRepository
db := newDB()
search := engine.NewSearch(artistRepository, albumRepository, mediaFileRepository, db)
importer := scanner.NewImporter(musicFolder, itunesScanner, mediaFileRepository, albumRepository, artistRepository, artistIndexRepository, playlistRepository, propertyRepository, search)
importer := scanner.NewImporter(musicFolder, itunesScanner, mediaFileRepository, albumRepository, artistRepository, artistIndexRepository, playlistRepository, propertyRepository)
app := NewApp(importer)
return app
}
@ -55,8 +51,7 @@ func CreateSubsonicAPIRouter(p persistence.ProviderIdentifier) *api.Router {
playlists := engine.NewPlaylists(itunesControl, playlistRepository, mediaFileRepository)
ratings := engine.NewRatings(itunesControl, mediaFileRepository, albumRepository, artistRepository)
scrobbler := engine.NewScrobbler(itunesControl, mediaFileRepository, nowPlayingRepository)
db := newDB()
search := engine.NewSearch(artistRepository, albumRepository, mediaFileRepository, db)
search := engine.NewSearch(artistRepository, albumRepository, mediaFileRepository)
router := api.NewRouter(browser, cover, listGenerator, playlists, ratings, scrobbler, search)
return router
}
@ -123,7 +118,7 @@ type Provider struct {
PropertyRepository domain.PropertyRepository
}
var allProviders = wire.NewSet(itunesbridge.NewItunesControl, engine.Set, scanner.Set, newDB, api.NewRouter, wire.FieldsOf(new(*Provider), "AlbumRepository", "ArtistRepository", "CheckSumRepository",
var allProviders = wire.NewSet(itunesbridge.NewItunesControl, engine.Set, scanner.Set, api.NewRouter, wire.FieldsOf(new(*Provider), "AlbumRepository", "ArtistRepository", "CheckSumRepository",
"ArtistIndexRepository", "MediaFileRepository", "MediaFolderRepository", "NowPlayingRepository",
"PlaylistRepository", "PropertyRepository"), createPersistenceProvider,
)
@ -136,7 +131,3 @@ func createPersistenceProvider(provider persistence.ProviderIdentifier) *Provide
return createLedisDBProvider()
}
}
func newDB() gomate.DB {
return ledis.NewEmbeddedDB(db_ledis.Db())
}

7
wire_injectors.go
View File

@ -11,8 +11,6 @@ import (
"github.com/cloudsonic/sonic-server/persistence/db_ledis"
"github.com/cloudsonic/sonic-server/persistence/db_sql"
"github.com/cloudsonic/sonic-server/scanner"
"github.com/deluan/gomate"
"github.com/deluan/gomate/ledis"
"github.com/google/wire"
)
@ -32,7 +30,6 @@ var allProviders = wire.NewSet(
itunesbridge.NewItunesControl,
engine.Set,
scanner.Set,
newDB,
api.NewRouter,
wire.FieldsOf(new(*Provider), "AlbumRepository", "ArtistRepository", "CheckSumRepository",
"ArtistIndexRepository", "MediaFileRepository", "MediaFolderRepository", "NowPlayingRepository",
@ -73,7 +70,3 @@ func createLedisDBProvider() *Provider {
wire.Struct(new(Provider), "*"),
))
}
func newDB() gomate.DB {
return ledis.NewEmbeddedDB(db_ledis.Db())
}