mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

WIP: Start scanner2, using charlievieth/fastwalk package

This commit is contained in:
Deluan 2023-12-16 15:15:27 -05:00
parent fd8bc1ae9c
commit 968df3c2a3
8 changed files with 115 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/root.go
View File

@ -87,15 +87,15 @@ func runNavidrome() {
func startServer(ctx context.Context) func() error {
return func() error {
a := CreateServer()
a.MountRouter("Native API", consts.URLPathNativeAPI, CreateNativeAPIRouter())
a.MountRouter("Subsonic API", consts.URLPathSubsonicAPI, CreateSubsonicAPIRouter())
a.MountRouter("Public Endpoints", consts.URLPathPublic, CreatePublicRouter())
a := CreateServer(ctx)
a.MountRouter("Native API", consts.URLPathNativeAPI, CreateNativeAPIRouter(ctx))
a.MountRouter("Subsonic API", consts.URLPathSubsonicAPI, CreateSubsonicAPIRouter(ctx))
a.MountRouter("Public Endpoints", consts.URLPathPublic, CreatePublicRouter(ctx))
if conf.Server.LastFM.Enabled {
a.MountRouter("LastFM Auth", consts.URLPathNativeAPI+"/lastfm", CreateLastFMRouter())
a.MountRouter("LastFM Auth", consts.URLPathNativeAPI+"/lastfm", CreateLastFMRouter(ctx))
}
if conf.Server.ListenBrainz.Enabled {
a.MountRouter("ListenBrainz Auth", consts.URLPathNativeAPI+"/listenbrainz", CreateListenBrainzRouter())
a.MountRouter("ListenBrainz Auth", consts.URLPathNativeAPI+"/listenbrainz", CreateListenBrainzRouter(ctx))
}
if conf.Server.Prometheus.Enabled {
// blocking call because takes <1ms but useful if fails
@ -120,7 +120,7 @@ func schedulePeriodicScan(ctx context.Context) func() error {
return nil
}
scanner := GetScanner()
scanner := GetScanner(ctx)
schedulerInstance := scheduler.GetInstance()
log.Info("Scheduling periodic scan", "schedule", schedule)

5
cmd/scan.go
View File

@ -24,8 +24,9 @@ var scanCmd = &cobra.Command{
}
func runScanner() {
scanner := GetScanner()
_ = scanner.RescanAll(context.Background(), fullRescan)
ctx := context.Background()
scanner := GetScanner(ctx)
_ = scanner.RescanAll(ctx, fullRescan)
if fullRescan {
log.Info("Finished full rescan")
} else {

2
cmd/signaler_unix.go
View File

@ -16,7 +16,7 @@ const triggerScanSignal = syscall.SIGUSR1
func startSignaler(ctx context.Context) func() error {
log.Info(ctx, "Starting signaler")
scanner := GetScanner()
scanner := GetScanner(ctx)
return func() error {
var sigChan = make(chan os.Signal, 1)

32
cmd/wire_gen.go
View File

@ -7,6 +7,7 @@
package cmd
import (
"context"
"github.com/google/wire"
"github.com/navidrome/navidrome/core"
"github.com/navidrome/navidrome/core/agents"
@ -18,6 +19,7 @@ import (
"github.com/navidrome/navidrome/db"
"github.com/navidrome/navidrome/persistence"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/scanner2"
"github.com/navidrome/navidrome/server"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/server/nativeapi"
@ -28,7 +30,7 @@ import (
// Injectors from wire_injectors.go:
func CreateServer() *server.Server {
func CreateServer(ctx context.Context) *server.Server {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
broker := events.GetBroker()
@ -36,7 +38,7 @@ func CreateServer() *server.Server {
return serverServer
}
func CreateNativeAPIRouter() *nativeapi.Router {
func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
share := core.NewShare(dataStore)
@ -45,7 +47,7 @@ func CreateNativeAPIRouter() *nativeapi.Router {
return router
}
func CreateSubsonicAPIRouter() *subsonic.Router {
func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
fileCache := artwork.GetImageCache()
@ -58,7 +60,7 @@ func CreateSubsonicAPIRouter() *subsonic.Router {
share := core.NewShare(dataStore)
archiver := core.NewArchiver(mediaStreamer, dataStore, share)
players := core.NewPlayers(dataStore)
scanner := GetScanner()
scanner := GetScanner(ctx)
broker := events.GetBroker()
playlists := core.NewPlaylists(dataStore)
playTracker := scrobbler.GetPlayTracker(dataStore, broker)
@ -66,7 +68,7 @@ func CreateSubsonicAPIRouter() *subsonic.Router {
return router
}
func CreatePublicRouter() *public.Router {
func CreatePublicRouter(ctx context.Context) *public.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
fileCache := artwork.GetImageCache()
@ -82,32 +84,24 @@ func CreatePublicRouter() *public.Router {
return router
}
func CreateLastFMRouter() *lastfm.Router {
func CreateLastFMRouter(ctx context.Context) *lastfm.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
router := lastfm.NewRouter(dataStore)
return router
}
func CreateListenBrainzRouter() *listenbrainz.Router {
func CreateListenBrainzRouter(ctx context.Context) *listenbrainz.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
router := listenbrainz.NewRouter(dataStore)
return router
}
func createScanner() scanner.Scanner {
func createScanner(ctx context.Context) scanner.Scanner {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
playlists := core.NewPlaylists(dataStore)
fileCache := artwork.GetImageCache()
fFmpeg := ffmpeg.New()
agentsAgents := agents.New(dataStore)
externalMetadata := core.NewExternalMetadata(dataStore, agentsAgents)
artworkArtwork := artwork.NewArtwork(dataStore, fileCache, fFmpeg, externalMetadata)
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
broker := events.GetBroker()
scannerScanner := scanner.New(dataStore, playlists, cacheWarmer, broker)
scannerScanner := scanner2.New(ctx, dataStore)
return scannerScanner
}
@ -121,9 +115,9 @@ var (
scannerInstance scanner.Scanner
)
func GetScanner() scanner.Scanner {
func GetScanner(ctx context.Context) scanner.Scanner {
onceScanner.Do(func() {
scannerInstance = createScanner()
scannerInstance = createScanner(ctx)
})
return scannerInstance
}

22
cmd/wire_injectors.go
View File

@ -3,6 +3,7 @@
package cmd
import (
"context"
"sync"
"github.com/google/wire"
@ -13,6 +14,7 @@ import (
"github.com/navidrome/navidrome/db"
"github.com/navidrome/navidrome/persistence"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/scanner2"
"github.com/navidrome/navidrome/server"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/server/nativeapi"
@ -33,39 +35,39 @@ var allProviders = wire.NewSet(
db.Db,
)
func CreateServer() *server.Server {
func CreateServer(ctx context.Context) *server.Server {
panic(wire.Build(
server.New,
allProviders,
))
}
func CreateNativeAPIRouter() *nativeapi.Router {
func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router {
panic(wire.Build(
allProviders,
))
}
func CreateSubsonicAPIRouter() *subsonic.Router {
func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router {
panic(wire.Build(
allProviders,
GetScanner,
))
}
func CreatePublicRouter() *public.Router {
func CreatePublicRouter(ctx context.Context) *public.Router {
panic(wire.Build(
allProviders,
))
}
func CreateLastFMRouter() *lastfm.Router {
func CreateLastFMRouter(ctx context.Context) *lastfm.Router {
panic(wire.Build(
allProviders,
))
}
func CreateListenBrainzRouter() *listenbrainz.Router {
func CreateListenBrainzRouter(ctx context.Context) *listenbrainz.Router {
panic(wire.Build(
allProviders,
))
@ -77,16 +79,16 @@ var (
scannerInstance scanner.Scanner
)
func GetScanner() scanner.Scanner {
func GetScanner(ctx context.Context) scanner.Scanner {
onceScanner.Do(func() {
scannerInstance = createScanner()
scannerInstance = createScanner(ctx)
})
return scannerInstance
}
func createScanner() scanner.Scanner {
func createScanner(ctx context.Context) scanner.Scanner {
panic(wire.Build(
allProviders,
scanner.New,
scanner2.New,
))
}

1
go.mod
View File

@ -6,6 +6,7 @@ require (
github.com/Masterminds/squirrel v1.5.4
github.com/RaveNoX/go-jsoncommentstrip v1.0.0
github.com/bradleyjkemp/cupaloy/v2 v2.8.0
github.com/charlievieth/fastwalk v1.0.1
github.com/deluan/rest v0.0.0-20211102003136-6260bc399cbf
github.com/deluan/sanitize v0.0.0-20230310221930-6e18967d9fc1
github.com/dexterlb/mpvipc v0.0.0-20230829142118-145d6eabdc37

1
go.sum
View File

@ -12,6 +12,7 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/charlievieth/fastwalk v1.0.1/go.mod h1:dryXgMJyGHbMrAmmnF0/EJNBbZaihlwcNud5IuGyogU=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

77
scanner2/scanner2.go Normal file
View File

@ -0,0 +1,77 @@
package scanner2
import (
"context"
"io/fs"
"github.com/charlievieth/fastwalk"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/utils/pl"
)
type scanner2 struct {
ctx context.Context
ds model.DataStore
}
func New(ctx context.Context, ds model.DataStore) scanner.Scanner {
return &scanner2{ctx: ctx, ds: ds}
}
func (s *scanner2) RescanAll(ctx context.Context, fullRescan bool) error {
libs, err := s.ds.Library(s.ctx).GetAll()
if err != nil {
return err
}
libsChan := pl.FromSlice(s.ctx, libs)
folderChan, folderErrChan := walkDirEntries(s.ctx, libsChan)
for folder := range folderChan {
log.Debug(s.ctx, "Scanner: Found folder", "folder", folder.Name())
}
// Wait for pipeline to end, return first error found
for err := range pl.Merge(ctx, folderErrChan) {
return err
}
return nil
}
func walkDirEntries(ctx context.Context, libsChan <-chan model.Library) (chan fastwalk.DirEntry, chan error) {
outputChannel := make(chan fastwalk.DirEntry)
errChannel := make(chan error)
go func() {
defer close(outputChannel)
defer close(errChannel)
errChan := pl.Sink(ctx, 1, libsChan, func(ctx context.Context, lib model.Library) error {
conf := &fastwalk.Config{Follow: true}
return fastwalk.Walk(conf, lib.Path, func(path string, d fs.DirEntry, err error) error {
if err != nil {
log.Error(ctx, "Scanner: Error walking path", "lib", lib.Name, "path", path, err)
}
if d.IsDir() {
outputChannel <- d.(fastwalk.DirEntry)
}
return nil
})
})
// Wait for pipeline to end, and forward any errors
for err := range pl.ReadOrDone(ctx, errChan) {
errChannel <- err
}
}()
return outputChannel, errChannel
}
func (s *scanner2) Status(context.Context) (*scanner.StatusInfo, error) {
return nil, nil
}
//nolint:unused
func (s *scanner2) doScan(ctx context.Context, lib model.Library, fullRescan bool, folders <-chan string) error {
return nil
}
var _ scanner.Scanner = (*scanner2)(nil)