mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge 2c9ccfe9e1 into 27875ba2dd

This commit is contained in:
Deluan Quintão 2024-04-28 16:40:11 +00:00 committed by GitHub
parent 27875ba2dd 2c9ccfe9e1
commit f89269aee5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
53 changed files with 1633 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/root.go
View File

@ -87,15 +87,15 @@ func runNavidrome() {
func startServer(ctx context.Context) func() error {
return func() error {
a := CreateServer(conf.Server.MusicFolder)
a.MountRouter("Native API", consts.URLPathNativeAPI, CreateNativeAPIRouter())
a.MountRouter("Subsonic API", consts.URLPathSubsonicAPI, CreateSubsonicAPIRouter())
a.MountRouter("Public Endpoints", consts.URLPathPublic, CreatePublicRouter())
a := CreateServer(ctx)
a.MountRouter("Native API", consts.URLPathNativeAPI, CreateNativeAPIRouter(ctx))
a.MountRouter("Subsonic API", consts.URLPathSubsonicAPI, CreateSubsonicAPIRouter(ctx))
a.MountRouter("Public Endpoints", consts.URLPathPublic, CreatePublicRouter(ctx))
if conf.Server.LastFM.Enabled {
a.MountRouter("LastFM Auth", consts.URLPathNativeAPI+"/lastfm", CreateLastFMRouter())
a.MountRouter("LastFM Auth", consts.URLPathNativeAPI+"/lastfm", CreateLastFMRouter(ctx))
}
if conf.Server.ListenBrainz.Enabled {
a.MountRouter("ListenBrainz Auth", consts.URLPathNativeAPI+"/listenbrainz", CreateListenBrainzRouter())
a.MountRouter("ListenBrainz Auth", consts.URLPathNativeAPI+"/listenbrainz", CreateListenBrainzRouter(ctx))
}
if conf.Server.Prometheus.Enabled {
// blocking call because takes <1ms but useful if fails
@ -120,7 +120,7 @@ func schedulePeriodicScan(ctx context.Context) func() error {
return nil
}
scanner := GetScanner()
scanner := GetScanner(ctx)
schedulerInstance := scheduler.GetInstance()
log.Info("Scheduling periodic scan", "schedule", schedule)

5
cmd/scan.go
View File

@ -24,8 +24,9 @@ var scanCmd = &cobra.Command{
}
func runScanner() {
scanner := GetScanner()
_ = scanner.RescanAll(context.Background(), fullRescan)
ctx := context.Background()
scanner := GetScanner(ctx)
_ = scanner.RescanAll(ctx, fullRescan)
if fullRescan {
log.Info("Finished full rescan")
} else {

2
cmd/signaler_unix.go
View File

@ -16,7 +16,7 @@ const triggerScanSignal = syscall.SIGUSR1
func startSignaler(ctx context.Context) func() error {
log.Info(ctx, "Starting signaler")
scanner := GetScanner()
scanner := GetScanner(ctx)
return func() error {
var sigChan = make(chan os.Signal, 1)

32
cmd/wire_gen.go
View File

@ -7,6 +7,7 @@
package cmd
import (
"context"
"github.com/google/wire"
"github.com/navidrome/navidrome/core"
"github.com/navidrome/navidrome/core/agents"
@ -18,6 +19,7 @@ import (
"github.com/navidrome/navidrome/db"
"github.com/navidrome/navidrome/persistence"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/scanner2"
"github.com/navidrome/navidrome/server"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/server/nativeapi"
@ -28,7 +30,7 @@ import (
// Injectors from wire_injectors.go:
func CreateServer(musicFolder string) *server.Server {
func CreateServer(ctx context.Context) *server.Server {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
broker := events.GetBroker()
@ -36,7 +38,7 @@ func CreateServer(musicFolder string) *server.Server {
return serverServer
}
func CreateNativeAPIRouter() *nativeapi.Router {
func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
share := core.NewShare(dataStore)
@ -45,7 +47,7 @@ func CreateNativeAPIRouter() *nativeapi.Router {
return router
}
func CreateSubsonicAPIRouter() *subsonic.Router {
func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
fileCache := artwork.GetImageCache()
@ -58,7 +60,7 @@ func CreateSubsonicAPIRouter() *subsonic.Router {
share := core.NewShare(dataStore)
archiver := core.NewArchiver(mediaStreamer, dataStore, share)
players := core.NewPlayers(dataStore)
scanner := GetScanner()
scanner := GetScanner(ctx)
broker := events.GetBroker()
playlists := core.NewPlaylists(dataStore)
playTracker := scrobbler.GetPlayTracker(dataStore, broker)
@ -66,7 +68,7 @@ func CreateSubsonicAPIRouter() *subsonic.Router {
return router
}
func CreatePublicRouter() *public.Router {
func CreatePublicRouter(ctx context.Context) *public.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
fileCache := artwork.GetImageCache()
@ -82,32 +84,24 @@ func CreatePublicRouter() *public.Router {
return router
}
func CreateLastFMRouter() *lastfm.Router {
func CreateLastFMRouter(ctx context.Context) *lastfm.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
router := lastfm.NewRouter(dataStore)
return router
}
func CreateListenBrainzRouter() *listenbrainz.Router {
func CreateListenBrainzRouter(ctx context.Context) *listenbrainz.Router {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
router := listenbrainz.NewRouter(dataStore)
return router
}
func createScanner() scanner.Scanner {
func createScanner(ctx context.Context) scanner.Scanner {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
playlists := core.NewPlaylists(dataStore)
fileCache := artwork.GetImageCache()
fFmpeg := ffmpeg.New()
agentsAgents := agents.New(dataStore)
externalMetadata := core.NewExternalMetadata(dataStore, agentsAgents)
artworkArtwork := artwork.NewArtwork(dataStore, fileCache, fFmpeg, externalMetadata)
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
broker := events.GetBroker()
scannerScanner := scanner.New(dataStore, playlists, cacheWarmer, broker)
scannerScanner := scanner2.New(ctx, dataStore)
return scannerScanner
}
@ -121,9 +115,9 @@ var (
scannerInstance scanner.Scanner
)
func GetScanner() scanner.Scanner {
func GetScanner(ctx context.Context) scanner.Scanner {
onceScanner.Do(func() {
scannerInstance = createScanner()
scannerInstance = createScanner(ctx)
})
return scannerInstance
}

22
cmd/wire_injectors.go
View File

@ -3,6 +3,7 @@
package cmd
import (
"context"
"sync"
"github.com/google/wire"
@ -13,6 +14,7 @@ import (
"github.com/navidrome/navidrome/db"
"github.com/navidrome/navidrome/persistence"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/scanner2"
"github.com/navidrome/navidrome/server"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/server/nativeapi"
@ -33,39 +35,39 @@ var allProviders = wire.NewSet(
db.Db,
)
func CreateServer(musicFolder string) *server.Server {
func CreateServer(ctx context.Context) *server.Server {
panic(wire.Build(
server.New,
allProviders,
))
}
func CreateNativeAPIRouter() *nativeapi.Router {
func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router {
panic(wire.Build(
allProviders,
))
}
func CreateSubsonicAPIRouter() *subsonic.Router {
func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router {
panic(wire.Build(
allProviders,
GetScanner,
))
}
func CreatePublicRouter() *public.Router {
func CreatePublicRouter(ctx context.Context) *public.Router {
panic(wire.Build(
allProviders,
))
}
func CreateLastFMRouter() *lastfm.Router {
func CreateLastFMRouter(ctx context.Context) *lastfm.Router {
panic(wire.Build(
allProviders,
))
}
func CreateListenBrainzRouter() *listenbrainz.Router {
func CreateListenBrainzRouter(ctx context.Context) *listenbrainz.Router {
panic(wire.Build(
allProviders,
))
@ -77,16 +79,16 @@ var (
scannerInstance scanner.Scanner
)
func GetScanner() scanner.Scanner {
func GetScanner(ctx context.Context) scanner.Scanner {
onceScanner.Do(func() {
scannerInstance = createScanner()
scannerInstance = createScanner(ctx)
})
return scannerInstance
}
func createScanner() scanner.Scanner {
func createScanner(ctx context.Context) scanner.Scanner {
panic(wire.Build(
allProviders,
scanner.New,
scanner2.New,
))
}

72
db/migration/20231212210020_add_library_table.go Normal file
View File

@ -0,0 +1,72 @@
package migrations
import (
"context"
"database/sql"
"fmt"
"github.com/navidrome/navidrome/conf"
"github.com/pressly/goose/v3"
)
func init() {
goose.AddMigrationContext(upAddLibraryTable, downAddLibraryTable)
}
func upAddLibraryTable(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, `
create table library (
id integer primary key autoincrement,
name varchar not null unique,
path varchar not null unique,
remote_path varchar null default '',
extractor varchar null default 'taglib',
last_scan_at datetime not null default '0000-00-00 00:00:00',
updated_at datetime not null default current_timestamp,
created_at datetime not null default current_timestamp
);`)
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, fmt.Sprintf(`
insert into library(id, name, path, extractor, last_scan_at) values(1, 'Music Library', '%s', '%s', current_timestamp);
delete from property where id like 'LastScan-%%';
`, conf.Server.MusicFolder, conf.Server.Scanner.Extractor))
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, `
alter table media_file add column library_id integer not null default 1
references library(id) on delete cascade;
alter table album add column library_id integer not null default 1
references library(id) on delete cascade;
create table if not exists library_artist
(
library_id integer not null default 1
references library(id)
on delete cascade,
artist_id varchar not null default null
references artist(id)
on delete cascade,
constraint library_artist_ux
unique (library_id, artist_id)
);
insert into library_artist(library_id, artist_id) select 1, id from artist;
`)
return err
}
func downAddLibraryTable(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, `
alter table media_file drop column library_id;
alter table album drop column library_id;
drop table library_artist;
drop table library;
`)
return err
}

73
db/migration/20231219003407_add_folder_table.go Normal file
View File

@ -0,0 +1,73 @@
package migrations
import (
"context"
"database/sql"
"github.com/pressly/goose/v3"
)
func init() {
goose.AddMigrationContext(upAddFolderTable, downAddFolderTable)
}
func upAddFolderTable(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, `
create table if not exists folder(
id varchar not null
primary key,
library_id integer not null
references library (id)
on delete cascade,
path varchar default '' not null,
name varchar default '' not null,
updated_at timestamp default current_timestamp not null,
created_at timestamp default current_timestamp not null,
parent_id varchar default '' not null
);
alter table media_file
add column folder_id varchar default "" not null;
alter table media_file
add column pid varchar default id not null;
alter table media_file
add column album_pid varchar default album_id not null;
alter table media_file
add column tags JSONB default '{}' not null;
create index if not exists media_file_folder_id_ix
on media_file (folder_id);
create unique index if not exists media_file_pid_ix
on media_file (pid);
create index if not exists media_file_album_pid_ix
on media_file (album_pid);
-- FIXME Needs to process current media_file.paths, creating folders as needed
create table if not exists tag(
id varchar not null primary key,
name varchar default '' not null,
value varchar default '' not null,
constraint tags_name_value_ux
unique (name, value)
);
create table if not exists item_tags(
item_id varchar not null,
item_type varchar not null,
tag_name varchar not null,
tag_id varchar not null,
constraint item_tags_ux
unique (item_id, item_type, tag_id)
);
create index if not exists item_tag_name_ix on item_tags(item_id, tag_name)
`)
return err
}
func downAddFolderTable(ctx context.Context, tx *sql.Tx) error {
// This code is executed when the migration is rolled back.
return nil
}

1
go.mod
View File

@ -21,6 +21,7 @@ require (
github.com/go-chi/cors v1.2.1
github.com/go-chi/httprate v0.9.0
github.com/go-chi/jwtauth/v5 v5.3.1
github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc
github.com/google/uuid v1.6.0
github.com/google/wire v0.6.0
github.com/hashicorp/go-multierror v1.1.1

2
go.sum
View File

@ -67,6 +67,8 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc h1:hd+uUVsB1vdxohPneMrhGH2YfQuH5hRIK9u4/XCeUtw=
github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc/go.mod h1:SL66SJVysrh7YbDCP9tH30b8a9o/N2HeiQNUm85EKhc=
github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk=
github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=

2
log/log.go
View File

@ -274,6 +274,8 @@ func addFields(logger *logrus.Entry, keyValuePairs []interface{}) *logrus.Entry
} else {
logger = logger.WithField(name, v.String())
}
case []string:
logger = logger.WithField(name, strings.Join(v, ","))
default:
logger = logger.WithField(name, v)
}

2
model/album.go
View File

@ -12,6 +12,8 @@ type Album struct {
Annotations `structs:"-"`
ID string `structs:"id" json:"id"`
PID string `structs:"pid" json:"pid"`
LibraryID string `structs:"library_id" json:"libraryId"`
Name string `structs:"name" json:"name"`
EmbedArtPath string `structs:"embed_art_path" json:"embedArtPath"`
ArtistID string `structs:"artist_id" json:"artistId"`

4
model/datastore.go
View File

@ -20,11 +20,13 @@ type ResourceRepository interface {
}
type DataStore interface {
Library(ctx context.Context) LibraryRepository
Folder(ctx context.Context) FolderRepository
Album(ctx context.Context) AlbumRepository
Artist(ctx context.Context) ArtistRepository
MediaFile(ctx context.Context) MediaFileRepository
MediaFolder(ctx context.Context) MediaFolderRepository
Genre(ctx context.Context) GenreRepository
Tag(ctx context.Context) TagRepository
Playlist(ctx context.Context) PlaylistRepository
PlayQueue(ctx context.Context) PlayQueueRepository
Transcoding(ctx context.Context) TranscodingRepository

54
model/folder.go Normal file
View File

@ -0,0 +1,54 @@
package model
import (
"crypto/md5"
"fmt"
"path/filepath"
"strings"
"time"
)
type Folder struct {
ID string `structs:"id"`
LibraryID int `structs:"library_id"`
Path string `structs:"path"`
Name string `structs:"name"`
ParentID string `structs:"parent_id"`
UpdateAt time.Time `structs:"updated_at"`
CreatedAt time.Time `structs:"created_at"`
}
func FolderID(lib Library, path string) string {
path = strings.TrimPrefix(path, lib.Path)
key := fmt.Sprintf("%d:%s", lib.ID, path)
return fmt.Sprintf("%x", md5.Sum([]byte(key)))
}
func NewFolder(lib Library, path string) *Folder {
id := FolderID(lib, path)
dir, name := filepath.Split(path)
dir = filepath.Clean(dir)
var parentID string
if dir == "." {
dir = ""
parentID = ""
} else {
parentID = FolderID(lib, dir)
}
return &Folder{
LibraryID: lib.ID,
ID: id,
Path: dir,
Name: name,
ParentID: parentID,
UpdateAt: time.Now(),
CreatedAt: time.Now(),
}
}
type FolderRepository interface {
Get(lib Library, path string) (*Folder, error)
GetAll(lib Library) ([]Folder, error)
GetLastUpdates(lib Library) (map[string]time.Time, error)
Put(lib Library, path string) error
Touch(lib Library, path string, t time.Time) error
}

32
model/library.go Normal file
View File

@ -0,0 +1,32 @@
package model
import (
"io/fs"
"os"
"time"
)
type Library struct {
ID int
Name string
Path string
RemotePath string
Extractor string
LastScanAt time.Time
UpdatedAt time.Time
CreatedAt time.Time
}
func (f Library) FS() fs.FS {
return os.DirFS(f.Path)
}
type Libraries []Library
type LibraryRepository interface {
Get(id int) (*Library, error)
Put(*Library) error
StoreMusicFolder() error
UpdateLastScan(id int, t time.Time) error
GetAll(...QueryOptions) (Libraries, error)
}

7
model/mediafile.go
View File

@ -21,6 +21,9 @@ type MediaFile struct {
Bookmarkable `structs:"-"`
ID string `structs:"id" json:"id"`
PID string `structs:"pid" json:"pid"`
LibraryID int `structs:"library_id" json:"libraryId"`
FolderID string `structs:"folder_id" json:"folderId"`
Path string `structs:"path" json:"path"`
Title string `structs:"title" json:"title"`
Album string `structs:"album" json:"album"`
@ -29,6 +32,7 @@ type MediaFile struct {
AlbumArtistID string `structs:"album_artist_id" json:"albumArtistId"`
AlbumArtist string `structs:"album_artist" json:"albumArtist"`
AlbumID string `structs:"album_id" json:"albumId"`
AlbumPID string `structs:"album_pid" json:"albumPid"`
HasCoverArt bool `structs:"has_cover_art" json:"hasCoverArt"`
TrackNumber int `structs:"track_number" json:"trackNumber"`
DiscNumber int `structs:"disc_number" json:"discNumber"`
@ -72,6 +76,8 @@ type MediaFile struct {
RgTrackGain float64 `structs:"rg_track_gain" json:"rgTrackGain"`
RgTrackPeak float64 `structs:"rg_track_peak" json:"rgTrackPeak"`
Tags Tags `structs:"tags" json:"tags,omitempty"` // All tags from the original file
CreatedAt time.Time `structs:"created_at" json:"createdAt"` // Time this entry was created in the DB
UpdatedAt time.Time `structs:"updated_at" json:"updatedAt"` // Time of file last update (mtime)
}
@ -271,4 +277,5 @@ type MediaFileRepository interface {
AnnotatedRepository
BookmarkableRepository
GetByFolder(folderID string) (MediaFiles, error)
}

23
model/mediafolder.go
View File

@ -1,23 +0,0 @@
package model
import (
"io/fs"
"os"
)
type MediaFolder struct {
ID int32
Name string
Path string
}
func (f MediaFolder) FS() fs.FS {
return os.DirFS(f.Path)
}
type MediaFolders []MediaFolder
type MediaFolderRepository interface {
Get(id int32) (*MediaFolder, error)
GetAll() (MediaFolders, error)
}

5
model/properties.go
View File

@ -1,10 +1,5 @@
package model
const (
// TODO Move other prop keys to here
PropLastScan = "LastScan"
)
type PropertyRepository interface {
Put(id string, value string) error
Get(id string) (string, error)

18
model/request/request.go
View File

@ -90,3 +90,21 @@ func ReverseProxyIpFrom(ctx context.Context) (string, bool) {
v, ok := ctx.Value(ReverseProxyIp).(string)
return v, ok
}
func AddValues(ctx, requestCtx context.Context) context.Context {
keys := []contextKey{
User,
Username,
Client,
Version,
Player,
Transcoding,
ClientUniqueId,
}
for _, key := range keys {
if v := requestCtx.Value(key); v != nil {
ctx = context.WithValue(ctx, key, v)
}
}
return ctx
}

59
model/tag.go Normal file
View File

@ -0,0 +1,59 @@
package model
import (
"crypto/md5"
"fmt"
"strings"
"github.com/navidrome/navidrome/consts"
)
type Tag struct {
ID string
Name string
Value string
}
type FlattenedTags []Tag
func (t Tag) String() string {
return fmt.Sprintf("%s=%s", t.Name, t.Value)
}
type Tags map[string][]string
func (t Tags) Values(name string) []string {
return t[name]
}
func (t Tags) Flatten(name string) FlattenedTags {
var tags FlattenedTags
for _, v := range t[name] {
tags = append(tags, NewTag(name, v))
}
return tags
}
func (t Tags) FlattenAll() FlattenedTags {
var tags FlattenedTags
for name, values := range t {
for _, v := range values {
tags = append(tags, NewTag(name, v))
}
}
return tags
}
func NewTag(name, value string) Tag {
name = strings.ToLower(name)
id := fmt.Sprintf("%x", md5.Sum([]byte(name+consts.Zwsp+strings.ToLower(value))))
return Tag{
ID: id,
Name: name,
Value: value,
}
}
type TagRepository interface {
Add(...Tag) error
}

2
persistence/album_repository.go
View File

@ -155,7 +155,7 @@ func (r *albumRepository) Get(id string) (*model.Album, error) {
}
func (r *albumRepository) Put(m *model.Album) error {
_, err := r.put(m.ID, &dbAlbum{Album: m})
_, err := r.put("pid", m.PID, &dbAlbum{Album: m})
if err != nil {
return err
}

2
persistence/artist_repository.go
View File

@ -95,7 +95,7 @@ func (r *artistRepository) Exists(id string) (bool, error) {
func (r *artistRepository) Put(a *model.Artist, colsToUpdate ...string) error {
a.FullText = getFullText(a.Name, a.SortArtistName)
dba := &dbArtist{Artist: a}
_, err := r.put(dba.ID, dba, colsToUpdate...)
_, err := r.put("id", dba.ID, dba, colsToUpdate...)
if err != nil {
return err
}

69
persistence/folder_repository.go Normal file
View File

@ -0,0 +1,69 @@
package persistence
import (
"context"
"time"
. "github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/model"
"github.com/pocketbase/dbx"
)
type folderRepository struct {
sqlRepository
}
func newFolderRepository(ctx context.Context, db dbx.Builder) model.FolderRepository {
r := &folderRepository{}
r.ctx = ctx
r.db = db
r.tableName = "folder"
return r
}
func (r folderRepository) Get(lib model.Library, path string) (*model.Folder, error) {
id := model.NewFolder(lib, path).ID
sq := r.newSelect().Where(Eq{"id": id})
var res model.Folder
err := r.queryOne(sq, res)
return &res, err
}
func (r folderRepository) GetAll(lib model.Library) ([]model.Folder, error) {
sq := r.newSelect().Columns("*").Where(Eq{"library_id": lib.ID})
var res []model.Folder
err := r.queryAll(sq, &res)
return res, err
}
func (r folderRepository) GetLastUpdates(lib model.Library) (map[string]time.Time, error) {
sq := r.newSelect().Columns("id", "updated_at").Where(Eq{"library_id": lib.ID})
var res []struct {
ID string
UpdatedAt time.Time
}
err := r.queryAll(sq, &res)
if err != nil {
return nil, err
}
m := make(map[string]time.Time, len(res))
for _, f := range res {
m[f.ID] = f.UpdatedAt
}
return m, nil
}
func (r folderRepository) Put(lib model.Library, path string) error {
folder := model.NewFolder(lib, path)
_, err := r.put("id", folder.ID, folder)
return err
}
func (r folderRepository) Touch(lib model.Library, path string, t time.Time) error {
id := model.FolderID(lib, path)
sq := Update(r.tableName).Set("updated_at", t).Where(Eq{"id": id})
_, err := r.executeSQL(sq)
return err
}
var _ model.FolderRepository = (*folderRepository)(nil)

73
persistence/library_repository.go Normal file
View File

@ -0,0 +1,73 @@
package persistence
import (
"context"
"time"
. "github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/model"
"github.com/pocketbase/dbx"
)
type libraryRepository struct {
sqlRepository
}
func NewLibraryRepository(ctx context.Context, db dbx.Builder) model.LibraryRepository {
r := &libraryRepository{}
r.ctx = ctx
r.db = db
r.tableName = "library"
return r
}
func (r *libraryRepository) Get(id int) (*model.Library, error) {
sq := r.newSelect().Columns("*").Where(Eq{"id": id})
var res model.Library
err := r.queryOne(sq, &res)
return &res, err
}
func (r *libraryRepository) Put(l *model.Library) error {
cols := map[string]any{
"name": l.Name,
"path": l.Path,
"remote_path": l.RemotePath,
"updated_at": time.Now(),
}
if l.ID != 0 {
cols["id"] = l.ID
}
sq := Insert(r.tableName).SetMap(cols).
Suffix(`ON CONFLICT(id) DO UPDATE set name = excluded.name, path = excluded.path,
remote_path = excluded.remote_path, updated_at = excluded.updated_at`)
_, err := r.executeSQL(sq)
return err
}
// TODO Remove this and the StoreMusicFolder method when we have a proper UI to add libraries
const hardCodedMusicFolderID = 1
func (r *libraryRepository) StoreMusicFolder() error {
sq := Update(r.tableName).Set("path", conf.Server.MusicFolder).Set("updated_at", time.Now()).
Set("extractor", conf.Server.Scanner.Extractor).Where(Eq{"id": hardCodedMusicFolderID})
_, err := r.executeSQL(sq)
return err
}
func (r *libraryRepository) UpdateLastScan(id int, t time.Time) error {
sq := Update(r.tableName).Set("last_scan_at", t).Where(Eq{"id": id})
_, err := r.executeSQL(sq)
return err
}
func (r *libraryRepository) GetAll(ops ...model.QueryOptions) (model.Libraries, error) {
sq := r.newSelect(ops...).Columns("*")
res := model.Libraries{}
err := r.queryAll(sq, &res)
return res, err
}
var _ model.LibraryRepository = (*libraryRepository)(nil)

127
persistence/mediafile_repository.go
View File

@ -2,6 +2,8 @@ package persistence
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
@ -21,7 +23,51 @@ type mediaFileRepository struct {
sqlRestful
}
func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepository {
type dbMediaFile struct {
*model.MediaFile `structs:",flatten"`
Tags string `structs:"-" json:"tags"`
}
func (m *dbMediaFile) PostScan() error {
if m.Tags == "" {
m.MediaFile.Tags = make(map[string][]string)
return nil
}
err := json.Unmarshal([]byte(m.Tags), &m.MediaFile.Tags)
if err != nil {
return err
}
// Map genres from tags
for _, g := range m.MediaFile.Tags.Flatten("genre") {
m.MediaFile.Genres = append(m.MediaFile.Genres, model.Genre{Name: g.Value, ID: g.ID})
}
return nil
}
func (m *dbMediaFile) PostMapArgs(args map[string]any) error {
if len(m.MediaFile.Tags) == 0 {
args["tags"] = "{}"
return nil
}
b, err := json.Marshal(m.MediaFile.Tags)
if err != nil {
return err
}
args["tags"] = string(b)
return nil
}
type dbMediaFiles []dbMediaFile
func (m *dbMediaFiles) toModels() model.MediaFiles {
res := make(model.MediaFiles, len(*m))
for i, mf := range *m {
res[i] = *mf.MediaFile
}
return res
}
func NewMediaFileRepository(ctx context.Context, db dbx.Builder) model.MediaFileRepository {
r := &mediaFileRepository{}
r.ctx = ctx
r.db = db
@ -50,7 +96,8 @@ func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepos
func (r *mediaFileRepository) CountAll(options ...model.QueryOptions) (int64, error) {
sql := r.newSelectWithAnnotation("media_file.id")
sql = r.withGenres(sql) // Required for filtering by genre
// FIXME Genres
//sql = r.withGenres(sql) // Required for filtering by genre
return r.count(sql, options...)
}
@ -59,66 +106,89 @@ func (r *mediaFileRepository) Exists(id string) (bool, error) {
}
func (r *mediaFileRepository) Put(m *model.MediaFile) error {
if m.ID == "" || m.PID == "" {
return errors.New("id and pid are required")
}
m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist,
m.SortTitle, m.SortAlbumName, m.SortArtistName, m.SortAlbumArtistName, m.DiscSubtitle)
_, err := r.put(m.ID, m)
_, err := r.put("pid", m.PID, &dbMediaFile{MediaFile: m})
if err != nil {
return err
}
return r.updateGenres(m.ID, r.tableName, m.Genres)
return r.updateTags(m.ID, m.Tags)
// FIXME Genres
//if err != nil {
// return err
//}
//return r.updateGenres(m.ID, r.tableName, m.Genres)
}
func (r *mediaFileRepository) selectMediaFile(options ...model.QueryOptions) SelectBuilder {
sql := r.newSelectWithAnnotation("media_file.id", options...).Columns("media_file.*")
sql = r.withBookmark(sql, "media_file.id")
if len(options) > 0 && options[0].Filters != nil {
s, _, _ := options[0].Filters.ToSql()
// If there's any reference of genre in the filter, joins with genre
if strings.Contains(s, "genre") {
sql = r.withGenres(sql)
// If there's no filter on genre_id, group the results by media_file.id
if !strings.Contains(s, "genre_id") {
sql = sql.GroupBy("media_file.id")
}
}
}
// FIXME Genres
//if len(options) > 0 && options[0].Filters != nil {
// s, _, _ := options[0].Filters.ToSql()
// // If there's any reference of genre in the filter, joins with genre
// if strings.Contains(s, "genre") {
// sql = r.withGenres(sql)
// // If there's no filter on genre_id, group the results by media_file.id
// if !strings.Contains(s, "genre_id") {
// sql = sql.GroupBy("media_file.id")
// }
// }
//}
return sql
}
func (r *mediaFileRepository) Get(id string) (*model.MediaFile, error) {
sel := r.selectMediaFile().Where(Eq{"media_file.id": id})
var res model.MediaFiles
var res dbMediaFiles
if err := r.queryAll(sel, &res); err != nil {
return nil, err
}
if len(res) == 0 {
return nil, model.ErrNotFound
}
err := r.loadMediaFileGenres(&res)
return &res[0], err
// FIXME Genres
//err := r.loadMediaFileGenres(&res)
return res[0].MediaFile, nil
}
func (r *mediaFileRepository) GetAll(options ...model.QueryOptions) (model.MediaFiles, error) {
sq := r.selectMediaFile(options...)
res := model.MediaFiles{}
var res dbMediaFiles
err := r.queryAll(sq, &res, options...)
if err != nil {
return nil, err
}
err = r.loadMediaFileGenres(&res)
return res, err
// FIXME Genres
//err = r.loadMediaFileGenres(&rows)
return res.toModels(), nil
}
func (r *mediaFileRepository) GetByFolder(folderID string) (model.MediaFiles, error) {
sq := r.newSelect().Columns("*").Where(Eq{"folder_id": folderID})
var res dbMediaFiles
err := r.queryAll(sq, &res)
if err != nil {
return nil, err
}
return res.toModels(), nil
}
func (r *mediaFileRepository) FindByPath(path string) (*model.MediaFile, error) {
sel := r.newSelect().Columns("*").Where(Like{"path": path})
var res model.MediaFiles
var res dbMediaFiles
if err := r.queryAll(sel, &res); err != nil {
return nil, err
}
if len(res) == 0 {
return nil, model.ErrNotFound
}
return &res[0], nil
return res[0].MediaFile, nil
}
func cleanPath(path string) string {
@ -144,9 +214,9 @@ func (r *mediaFileRepository) FindAllByPath(path string) (model.MediaFiles, erro
sel := r.newSelect().Columns("*", "item NOT GLOB '*"+string(os.PathSeparator)+"*' AS isLast").
Where(Eq{"isLast": 1}).FromSelect(sel0, "sel0")
res := model.MediaFiles{}
res := dbMediaFiles{}
err := r.queryAll(sel, &res)
return res, err
return res.toModels(), err
}
// FindPathsRecursively returns a list of all subfolders of basePath, recursively
@ -195,13 +265,14 @@ func (r *mediaFileRepository) removeNonAlbumArtistIds() error {
}
func (r *mediaFileRepository) Search(q string, offset int, size int) (model.MediaFiles, error) {
results := model.MediaFiles{}
results := dbMediaFiles{}
err := r.doSearch(q, offset, size, &results, "title")
if err != nil {
return nil, err
}
err = r.loadMediaFileGenres(&results)
return results, err
// FIXME Genres
//err = r.loadMediaFileGenres(&results)
return results.toModels(), err
}
func (r *mediaFileRepository) Count(options ...rest.QueryOptions) (int64, error) {

37
persistence/mediafolders_repository.go
View File

@ -1,37 +0,0 @@
package persistence
import (
"context"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/model"
"github.com/pocketbase/dbx"
)
type mediaFolderRepository struct {
ctx context.Context
}
func NewMediaFolderRepository(ctx context.Context, _ dbx.Builder) model.MediaFolderRepository {
return &mediaFolderRepository{ctx}
}
func (r *mediaFolderRepository) Get(id int32) (*model.MediaFolder, error) {
mediaFolder := hardCoded()
return &mediaFolder, nil
}
func (*mediaFolderRepository) GetAll() (model.MediaFolders, error) {
mediaFolder := hardCoded()
result := make(model.MediaFolders, 1)
result[0] = mediaFolder
return result, nil
}
func hardCoded() model.MediaFolder {
mediaFolder := model.MediaFolder{ID: 0, Path: conf.Server.MusicFolder}
mediaFolder.Name = "Music Library"
return mediaFolder
}
var _ model.MediaFolderRepository = (*mediaFolderRepository)(nil)

12
persistence/persistence.go
View File

@ -31,14 +31,22 @@ func (s *SQLStore) MediaFile(ctx context.Context) model.MediaFileRepository {
return NewMediaFileRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) MediaFolder(ctx context.Context) model.MediaFolderRepository {
return NewMediaFolderRepository(ctx, s.getDBXBuilder())
func (s *SQLStore) Library(ctx context.Context) model.LibraryRepository {
return NewLibraryRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) Folder(ctx context.Context) model.FolderRepository {
return newFolderRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) Genre(ctx context.Context) model.GenreRepository {
return NewGenreRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) Tag(ctx context.Context) model.TagRepository {
return NewTagRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) PlayQueue(ctx context.Context) model.PlayQueueRepository {
return NewPlayQueueRepository(ctx, s.getDBXBuilder())
}

8
persistence/persistence_suite_test.go
View File

@ -60,10 +60,10 @@ var (
)
var (
songDayInALife = model.MediaFile{ID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Genre: "Rock", Genres: model.Genres{genreRock}, Path: P("/beatles/1/sgt/a day.mp3"), FullText: " a beatles day in life peppers sgt the"}
songComeTogether = model.MediaFile{ID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Genre: "Rock", Genres: model.Genres{genreRock}, Path: P("/beatles/1/come together.mp3"), FullText: " abbey beatles come road the together"}
songRadioactivity = model.MediaFile{ID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Genre: "Electronic", Genres: model.Genres{genreElectronic}, Path: P("/kraft/radio/radio.mp3"), FullText: " kraftwerk radioactivity"}
songAntenna = model.MediaFile{ID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk",
songDayInALife = model.MediaFile{ID: "1001", PID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Genre: "Rock", Genres: model.Genres{genreRock}, Path: P("/beatles/1/sgt/a day.mp3"), FullText: " a beatles day in life peppers sgt the"}
songComeTogether = model.MediaFile{ID: "1002", PID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Genre: "Rock", Genres: model.Genres{genreRock}, Path: P("/beatles/1/come together.mp3"), FullText: " abbey beatles come road the together"}
songRadioactivity = model.MediaFile{ID: "1003", PID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Genre: "Electronic", Genres: model.Genres{genreElectronic}, Path: P("/kraft/radio/radio.mp3"), FullText: " kraftwerk radioactivity"}
songAntenna = model.MediaFile{ID: "1004", PID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk",
AlbumID: "103", Genre: "Electronic", Genres: model.Genres{genreElectronic, genreRock},
Path: P("/kraft/radio/antenna.mp3"), FullText: " antenna kraftwerk",
RgAlbumGain: 1.0, RgAlbumPeak: 2.0, RgTrackGain: 3.0, RgTrackPeak: 4.0,

6
persistence/player_repository.go
View File

@ -27,7 +27,7 @@ func NewPlayerRepository(ctx context.Context, db dbx.Builder) model.PlayerReposi
}
func (r *playerRepository) Put(p *model.Player) error {
_, err := r.put(p.ID, p)
_, err := r.put("id", p.ID, p)
return err
}
@ -102,7 +102,7 @@ func (r *playerRepository) Save(entity interface{}) (string, error) {
if !r.isPermitted(t) {
return "", rest.ErrPermissionDenied
}
id, err := r.put(t.ID, t)
id, err := r.put("id", t.ID, t)
if errors.Is(err, model.ErrNotFound) {
return "", rest.ErrNotFound
}
@ -115,7 +115,7 @@ func (r *playerRepository) Update(id string, entity interface{}, cols ...string)
if !r.isPermitted(t) {
return rest.ErrPermissionDenied
}
_, err := r.put(id, t, cols...)
_, err := r.put("id", id, t, cols...)
if errors.Is(err, model.ErrNotFound) {
return rest.ErrNotFound
}

4
persistence/playlist_repository.go
View File

@ -118,7 +118,7 @@ func (r *playlistRepository) Put(p *model.Playlist) error {
}
pls.UpdatedAt = time.Now()
id, err := r.put(pls.ID, pls)
id, err := r.put("id", pls.ID, pls)
if err != nil {
return err
}
@ -417,7 +417,7 @@ func (r *playlistRepository) Update(id string, entity interface{}, cols ...strin
}
pls.ID = id
pls.UpdatedAt = time.Now()
_, err = r.put(id, pls, append(cols, "updatedAt")...)
_, err = r.put("id", id, pls, append(cols, "updatedAt")...)
if errors.Is(err, model.ErrNotFound) {
return rest.ErrNotFound
}

2
persistence/playqueue_repository.go
View File

@ -47,7 +47,7 @@ func (r *playQueueRepository) Store(q *model.PlayQueue) error {
pq.CreatedAt = time.Now()
}
pq.UpdatedAt = time.Now()
_, err = r.put(pq.ID, pq)
_, err = r.put("id", pq.ID, pq)
if err != nil {
log.Error(r.ctx, "Error saving playqueue", "user", u.UserName, err)
return err

4
persistence/share_repository.go
View File

@ -138,7 +138,7 @@ func (r *shareRepository) Update(id string, entity interface{}, cols ...string)
s.ID = id
s.UpdatedAt = time.Now()
cols = append(cols, "updated_at")
_, err := r.put(id, s, cols...)
_, err := r.put("id", id, s, cols...)
if errors.Is(err, model.ErrNotFound) {
return rest.ErrNotFound
}
@ -154,7 +154,7 @@ func (r *shareRepository) Save(entity interface{}) (string, error) {
}
s.CreatedAt = time.Now()
s.UpdatedAt = time.Now()
id, err := r.put(s.ID, s)
id, err := r.put("id", s.ID, s)
if errors.Is(err, model.ErrNotFound) {
return "", rest.ErrNotFound
}

16
persistence/sql_base_repository.go
View File

@ -237,10 +237,10 @@ func (r sqlRepository) count(countQuery SelectBuilder, options ...model.QueryOpt
return res.Count, err
}
func (r sqlRepository) put(id string, m interface{}, colsToUpdate ...string) (newId string, err error) {
func (r sqlRepository) put(idCol string, idVal string, m interface{}, colsToUpdate ...string) (newId string, err error) {
values, _ := toSQLArgs(m)
// If there's an ID, try to update first
if id != "" {
if idVal != "" {
updateValues := map[string]interface{}{}
// This is a map of the columns that need to be updated, if specified
@ -255,23 +255,23 @@ func (r sqlRepository) put(id string, m interface{}, colsToUpdate ...string) (ne
}
delete(updateValues, "created_at")
update := Update(r.tableName).Where(Eq{"id": id}).SetMap(updateValues)
update := Update(r.tableName).Where(Eq{idCol: idVal}).SetMap(updateValues)
count, err := r.executeSQL(update)
if err != nil {
return "", err
}
if count > 0 {
return id, nil
return idVal, nil
}
}
// If it does not have an ID OR the ID was not found (when it is a new record with predefined id)
if id == "" {
id = uuid.NewString()
values["id"] = id
if idVal == "" {
idVal = uuid.NewString()
values[idCol] = idVal
}
insert := Insert(r.tableName).SetMap(values)
_, err = r.executeSQL(insert)
return id, err
return idVal, err
}
func (r sqlRepository) delete(cond Sqlizer) error {

55
persistence/tag_repository.go Normal file
View File

@ -0,0 +1,55 @@
package persistence
import (
"context"
. "github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/slice"
"github.com/pocketbase/dbx"
)
type tagRepository struct {
sqlRepository
}
func NewTagRepository(ctx context.Context, db dbx.Builder) model.TagRepository {
r := &tagRepository{}
r.ctx = ctx
r.db = db
r.tableName = "tag"
return r
}
func (r *tagRepository) Add(tags ...model.Tag) error {
return slice.RangeByChunks(tags, 200, func(chunk []model.Tag) error {
sq := Insert(r.tableName).Columns("id", "name", "value").
Suffix("on conflict (id) do nothing")
for _, t := range chunk {
sq = sq.Values(t.ID, t.Name, t.Value)
}
_, err := r.executeSQL(sq)
return err
})
}
func (r *sqlRepository) updateTags(itemID string, tags model.Tags) error {
sqd := Delete("item_tags").Where(Eq{"item_id": itemID, "item_type": r.tableName})
_, err := r.executeSQL(sqd)
if err != nil {
return err
}
if len(tags) == 0 {
return nil
}
sqi := Insert("item_tags").Columns("item_id", "item_type", "tag_name", "tag_id").
Suffix("on conflict (item_id, item_type, tag_id) do nothing")
for name, values := range tags {
for _, value := range values {
tag := model.NewTag(name, value)
sqi = sqi.Values(itemID, r.tableName, tag.Name, tag.ID)
}
}
_, err = r.executeSQL(sqi)
return err
}

6
persistence/transcoding_repository.go
View File

@ -42,7 +42,7 @@ func (r *transcodingRepository) FindByFormat(format string) (*model.Transcoding,
}
func (r *transcodingRepository) Put(t *model.Transcoding) error {
_, err := r.put(t.ID, t)
_, err := r.put("id", t.ID, t)
return err
}
@ -71,7 +71,7 @@ func (r *transcodingRepository) NewInstance() interface{} {
func (r *transcodingRepository) Save(entity interface{}) (string, error) {
t := entity.(*model.Transcoding)
id, err := r.put(t.ID, t)
id, err := r.put("id", t.ID, t)
if errors.Is(err, model.ErrNotFound) {
return "", rest.ErrNotFound
}
@ -81,7 +81,7 @@ func (r *transcodingRepository) Save(entity interface{}) (string, error) {
func (r *transcodingRepository) Update(id string, entity interface{}, cols ...string) error {
t := entity.(*model.Transcoding)
t.ID = id
_, err := r.put(id, t)
_, err := r.put("id", id, t)
if errors.Is(err, model.ErrNotFound) {
return rest.ErrNotFound
}

83
scanner/metadata/metadata.go
View File

@ -222,6 +222,7 @@ func (t Tags) Channels() int { return t.getInt("channels") }
func (t Tags) ModificationTime() time.Time { return t.fileInfo.ModTime() }
func (t Tags) Size() int64 { return t.fileInfo.Size() }
func (t Tags) FilePath() string { return t.filePath }
func (t Tags) Folder() string { return path.Dir(t.filePath) }
func (t Tags) Suffix() string { return strings.ToLower(strings.TrimPrefix(path.Ext(t.filePath), ".")) }
func (t Tags) BirthTime() time.Time {
if ts := times.Get(t.fileInfo); ts.HasBirthTime() {
@ -230,7 +231,7 @@ func (t Tags) BirthTime() time.Time {
return time.Now()
}
// ReplayGain Properties
// ReplayGain Properties TODO: check rg_* tags
func (t Tags) RGAlbumGain() float64 { return t.getGainValue("replaygain_album_gain") }
func (t Tags) RGAlbumPeak() float64 { return t.getPeakValue("replaygain_album_peak") }
@ -384,3 +385,83 @@ func (t Tags) getFloat(tagNames ...string) float64 {
}
return value
}
// We exclude all tags that are already first-class citizens in the model
var excludedTags = map[string]struct{}{
"duration": {},
"length": {},
"tlen": {},
"bitrate": {},
"channels": {},
"has_picture": {},
"apic": {},
"bpm": {},
"tbpm": {},
"tipl": {},
"tcom": {},
"tit2": {},
"talb": {},
"tcon": {},
"tpe1": {},
"tpe2": {},
"title": {},
"album": {},
"artist": {},
"artists": {},
"albumartist": {},
"albumartists": {},
"composer": {},
"track": {},
"tracknumber": {},
"tracktotal": {},
"totaltracks": {},
"disc": {},
"discnumber": {},
"disctotal": {},
"totaldiscs": {},
"year": {},
"date": {},
"originaldate": {},
"releasedate": {},
"comm": {},
"comment": {},
"comment:itunnorm": {},
"uslt": {},
}
// Also exclude any tag that starts with one of these prefixes
var excludedPrefixes = []string{
"musicbrainz",
"replaygain",
"sort",
"lyrics",
}
func isExcludedTag(tagName string) bool {
if _, ok := excludedTags[tagName]; ok {
return true
}
for _, prefix := range excludedPrefixes {
if strings.HasPrefix(tagName, prefix) {
return true
}
}
return false
}
func (t Tags) ModelTags() model.Tags {
models := model.Tags{}
for tagName, values := range t.Tags {
if isExcludedTag(tagName) {
continue
}
for _, value := range values {
if value == "" {
continue
}
models[tagName] = append(models[tagName], value)
}
}
return models
}

1
scanner/metadata/taglib/taglib.go
View File

@ -50,6 +50,7 @@ func (e *Extractor) extractMetadata(filePath string) (metadata.ParsedTags, error
if duration := float64(millis) / 1000.0; duration > 0 {
tags["duration"] = []string{strconv.FormatFloat(duration, 'f', 2, 32)}
}
delete(tags, "lengthinmilliseconds")
}
// Adjust some ID3 tags
parseTIPL(tags)

6
scanner/metadata/taglib/taglib_wrapper.go
View File

@ -126,7 +126,11 @@ func do_put_map(id C.ulong, key string, val *C.char) {
}
/*
As I'm working on the new scanner, I see that the `properties` from TagLib is ill-suited to extract multi-valued ID3 frames. I'll have to change the way we do it for ID3, probably by sending the raw frames to Go and mapping there, instead of relying on the auto-mapped `properties`. I think this would reduce our reliance on C++, while also giving us more flexibility, including parsing the USLT / SYLT frames in Go
TODO: Validate this assumption:
"As I'm working on the new scanner, I see that the `properties` from TagLib is ill-suited to extract multi-valued
ID3 frames. I'll have to change the way we do it for ID3, probably by sending the raw frames to Go and mapping there,
instead of relying on the auto-mapped `properties`. I think this would reduce our reliance on C++, while also giving
us more flexibility, including parsing the USLT / SYLT frames in Go (https://github.com/n10v/id3v2/pull/64)"
*/
//export go_map_put_int

76
scanner/scanner.go
View File

@ -3,11 +3,10 @@ package scanner
import (
"context"
"errors"
"fmt"
"strconv"
"sync"
"time"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/core"
"github.com/navidrome/navidrome/core/artwork"
"github.com/navidrome/navidrome/log"
@ -17,11 +16,10 @@ import (
type Scanner interface {
RescanAll(ctx context.Context, fullRescan bool) error
Status(mediaFolder string) (*StatusInfo, error)
Status(context.Context) (*StatusInfo, error)
}
type StatusInfo struct {
MediaFolder string
Scanning bool
LastScan time.Time
Count uint32
@ -41,7 +39,10 @@ type FolderScanner interface {
var isScanning sync.Mutex
type scanner struct {
once sync.Once
folders map[string]FolderScanner
lastScans map[string]time.Time
libIds map[string]int
status map[string]*scanStatus
lock *sync.RWMutex
ds model.DataStore
@ -63,55 +64,56 @@ func New(ds model.DataStore, playlists core.Playlists, cacheWarmer artwork.Cache
pls: playlists,
broker: broker,
folders: map[string]FolderScanner{},
lastScans: map[string]time.Time{},
libIds: map[string]int{},
status: map[string]*scanStatus{},
lock: &sync.RWMutex{},
cacheWarmer: cacheWarmer,
}
s.loadFolders()
return s
}
func (s *scanner) rescan(ctx context.Context, mediaFolder string, fullRescan bool) error {
folderScanner := s.folders[mediaFolder]
func (s *scanner) rescan(ctx context.Context, library string, fullRescan bool) error {
folderScanner := s.folders[library]
start := time.Now()
s.setStatusStart(mediaFolder)
defer s.setStatusEnd(mediaFolder, start)
s.setStatusStart(library)
defer s.setStatusEnd(library, start)
lastModifiedSince := time.Time{}
if !fullRescan {
lastModifiedSince = s.getLastModifiedSince(ctx, mediaFolder)
log.Debug("Scanning folder", "folder", mediaFolder, "lastModifiedSince", lastModifiedSince)
lastModifiedSince = s.lastScans[library]
log.Debug("Scanning folder", "folder", library, "lastModifiedSince", lastModifiedSince)
} else {
log.Debug("Scanning folder (full scan)", "folder", mediaFolder)
log.Debug("Scanning folder (full scan)", "folder", library)
}
progress, cancel := s.startProgressTracker(mediaFolder)
progress, cancel := s.startProgressTracker(library)
defer cancel()
changeCount, err := folderScanner.Scan(ctx, lastModifiedSince, progress)
if err != nil {
log.Error("Error importing MediaFolder", "folder", mediaFolder, err)
log.Error("Error scanning Library", "folder", library, err)
}
if changeCount > 0 {
log.Debug(ctx, "Detected changes in the music folder. Sending refresh event",
"folder", mediaFolder, "changeCount", changeCount)
"folder", library, "changeCount", changeCount)
// Don't use real context, forcing a refresh in all open windows, including the one that triggered the scan
s.broker.SendMessage(context.Background(), &events.RefreshResource{})
}
s.updateLastModifiedSince(mediaFolder, start)
s.updateLastModifiedSince(library, start)
return err
}
func (s *scanner) startProgressTracker(mediaFolder string) (chan uint32, context.CancelFunc) {
func (s *scanner) startProgressTracker(library string) (chan uint32, context.CancelFunc) {
ctx, cancel := context.WithCancel(context.Background())
progress := make(chan uint32, 100)
go func() {
s.broker.SendMessage(ctx, &events.ScanStatus{Scanning: true, Count: 0, FolderCount: 0})
defer func() {
if status, ok := s.getStatus(mediaFolder); ok {
if status, ok := s.getStatus(library); ok {
s.broker.SendMessage(ctx, &events.ScanStatus{
Scanning: false,
Count: int64(status.fileCount),
@ -127,7 +129,7 @@ func (s *scanner) startProgressTracker(mediaFolder string) (chan uint32, context
if count == 0 {
continue
}
totalFolders, totalFiles := s.incStatusCounter(mediaFolder, count)
totalFolders, totalFiles := s.incStatusCounter(library, count)
s.broker.SendMessage(ctx, &events.ScanStatus{
Scanning: true,
Count: int64(totalFiles),
@ -179,6 +181,8 @@ func (s *scanner) setStatusEnd(folder string, lastUpdate time.Time) {
func (s *scanner) RescanAll(ctx context.Context, fullRescan bool) error {
ctx = context.WithoutCancel(ctx)
s.once.Do(s.loadFolders)
if !isScanning.TryLock() {
log.Debug(ctx, "Scanner already running, ignoring request for rescan.")
return ErrAlreadyScanning
@ -198,13 +202,14 @@ func (s *scanner) RescanAll(ctx context.Context, fullRescan bool) error {
core.WriteAfterScanMetrics(ctx, s.ds, true)
return nil
}
func (s *scanner) Status(mediaFolder string) (*StatusInfo, error) {
status, ok := s.getStatus(mediaFolder)
func (s *scanner) Status(context.Context) (*StatusInfo, error) {
s.once.Do(s.loadFolders)
status, ok := s.getStatus(conf.Server.MusicFolder)
if !ok {
return nil, errors.New("mediaFolder not found")
return nil, errors.New("library not found")
}
return &StatusInfo{
MediaFolder: mediaFolder,
Scanning: status.active,
LastScan: status.lastUpdate,
Count: status.fileCount,
@ -212,40 +217,31 @@ func (s *scanner) Status(mediaFolder string) (*StatusInfo, error) {
}, nil
}
func (s *scanner) getLastModifiedSince(ctx context.Context, folder string) time.Time {
ms, err := s.ds.Property(ctx).Get(model.PropLastScan + "-" + folder)
if err != nil {
return time.Time{}
}
if ms == "" {
return time.Time{}
}
i, _ := strconv.ParseInt(ms, 10, 64)
return time.Unix(0, i*int64(time.Millisecond))
}
func (s *scanner) updateLastModifiedSince(folder string, t time.Time) {
millis := t.UnixNano() / int64(time.Millisecond)
if err := s.ds.Property(context.TODO()).Put(model.PropLastScan+"-"+folder, fmt.Sprint(millis)); err != nil {
id := s.libIds[folder]
if err := s.ds.Library(context.Background()).UpdateLastScan(id, t); err != nil {
log.Error("Error updating DB after scan", err)
}
s.lastScans[folder] = t
}
func (s *scanner) loadFolders() {
ctx := context.TODO()
fs, _ := s.ds.MediaFolder(ctx).GetAll()
fs, _ := s.ds.Library(ctx).GetAll()
for _, f := range fs {
log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path)
s.folders[f.Path] = s.newScanner(f)
s.lastScans[f.Path] = f.LastScanAt
s.libIds[f.Path] = f.ID
s.status[f.Path] = &scanStatus{
active: false,
fileCount: 0,
folderCount: 0,
lastUpdate: s.getLastModifiedSince(ctx, f.Path),
lastUpdate: f.LastScanAt,
}
}
}
func (s *scanner) newScanner(f model.MediaFolder) FolderScanner {
func (s *scanner) newScanner(f model.Library) FolderScanner {
return NewTagScanner(f.Path, s.ds, s.pls, s.cacheWarmer)
}

29
scanner2/folder.go Normal file
View File

@ -0,0 +1,29 @@
package scanner2
import (
"io/fs"
"time"
"github.com/navidrome/navidrome/model"
)
type folderEntry struct {
scanCtx *scanContext
path string // Full path
id string // DB ID
updTime time.Time // From DB
modTime time.Time // From FS
audioFiles map[string]fs.DirEntry
imageFiles map[string]fs.DirEntry
playlists []fs.DirEntry
imagesUpdatedAt time.Time
tracks model.MediaFiles
albums model.Albums
artists model.Artists
tags model.FlattenedTags
missingTracks model.MediaFiles
}
func (f *folderEntry) isOutdated() bool {
return f.updTime.Before(f.modTime)
}

200
scanner2/mapping.go Normal file
View File

@ -0,0 +1,200 @@
package scanner2
import (
"crypto/md5"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/deluan/sanitize"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/scanner/metadata"
"github.com/navidrome/navidrome/utils"
)
type mediaFileMapper struct {
entry *folderEntry
}
func newMediaFileMapper(entry *folderEntry) *mediaFileMapper {
return &mediaFileMapper{
entry: entry,
}
}
func (s mediaFileMapper) toMediaFile(md metadata.Tags) model.MediaFile {
mf := &model.MediaFile{}
mf.ID = s.trackID(md)
mf.Year, mf.Date, mf.OriginalYear, mf.OriginalDate, mf.ReleaseYear, mf.ReleaseDate = s.mapDates(md)
mf.Title = s.mapTrackTitle(md)
mf.Album = md.Album()
mf.AlbumID = s.albumID(md, mf.ReleaseDate)
mf.Album = s.mapAlbumName(md)
mf.ArtistID = s.artistID(md)
mf.Artist = s.mapArtistName(md)
mf.AlbumArtistID = s.albumArtistID(md)
mf.AlbumArtist = s.mapAlbumArtistName(md)
mf.Genre, mf.Genres = s.mapGenres(md.Genres())
mf.Compilation = md.Compilation()
mf.TrackNumber, _ = md.TrackNumber()
mf.DiscNumber, _ = md.DiscNumber()
mf.DiscSubtitle = md.DiscSubtitle()
mf.Duration = md.Duration()
mf.BitRate = md.BitRate()
mf.Channels = md.Channels()
mf.Path = md.FilePath()
mf.Suffix = md.Suffix()
mf.Size = md.Size()
mf.HasCoverArt = md.HasPicture()
mf.SortTitle = md.SortTitle()
mf.SortAlbumName = md.SortAlbum()
mf.SortArtistName = md.SortArtist()
mf.SortAlbumArtistName = md.SortAlbumArtist()
mf.OrderTitle = strings.TrimSpace(sanitize.Accents(mf.Title))
mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album)
mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist)
mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist)
mf.CatalogNum = md.CatalogNum()
mf.MbzRecordingID = md.MbzRecordingID()
mf.MbzReleaseTrackID = md.MbzReleaseTrackID()
mf.MbzAlbumID = md.MbzAlbumID()
mf.MbzArtistID = md.MbzArtistID()
mf.MbzAlbumArtistID = md.MbzAlbumArtistID()
mf.MbzAlbumType = md.MbzAlbumType()
mf.MbzAlbumComment = md.MbzAlbumComment()
mf.RgAlbumGain = md.RGAlbumGain()
mf.RgAlbumPeak = md.RGAlbumPeak()
mf.RgTrackGain = md.RGTrackGain()
mf.RgTrackPeak = md.RGTrackPeak()
mf.Comment = utils.SanitizeText(md.Comment())
mf.Lyrics = md.Lyrics()
mf.Bpm = md.Bpm()
mf.CreatedAt = md.BirthTime()
mf.UpdatedAt = md.ModificationTime()
mf.Tags = md.ModelTags()
mf.FolderID = s.entry.id
mf.LibraryID = s.entry.scanCtx.lib.ID
mf.PID = mf.ID
return *mf
}
func sanitizeFieldForSorting(originalValue string) string {
v := strings.TrimSpace(sanitize.Accents(originalValue))
return utils.NoArticle(v)
}
func (s mediaFileMapper) mapTrackTitle(md metadata.Tags) string {
if md.Title() == "" {
s := strings.TrimPrefix(md.FilePath(), s.entry.path+string(os.PathSeparator))
e := filepath.Ext(s)
return strings.TrimSuffix(s, e)
}
return md.Title()
}
func (s mediaFileMapper) mapAlbumArtistName(md metadata.Tags) string {
switch {
case md.AlbumArtist() != "":
return md.AlbumArtist()
case md.Compilation():
return consts.VariousArtists
case md.Artist() != "":
return md.Artist()
default:
return consts.UnknownArtist
}
}
func (s mediaFileMapper) mapArtistName(md metadata.Tags) string {
if md.Artist() != "" {
return md.Artist()
}
return consts.UnknownArtist
}
func (s mediaFileMapper) mapAlbumName(md metadata.Tags) string {
name := md.Album()
if name == "" {
return consts.UnknownAlbum
}
return name
}
func (s mediaFileMapper) trackID(md metadata.Tags) string {
return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
}
func (s mediaFileMapper) albumID(md metadata.Tags, releaseDate string) string {
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
if !conf.Server.Scanner.GroupAlbumReleases {
if len(releaseDate) != 0 {
albumPath = fmt.Sprintf("%s\\%s", albumPath, releaseDate)
}
}
return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
}
func (s mediaFileMapper) artistID(md metadata.Tags) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
}
func (s mediaFileMapper) albumArtistID(md metadata.Tags) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
}
func (s mediaFileMapper) mapGenres(genres []string) (string, model.Genres) {
var result model.Genres
unique := map[string]struct{}{}
var all []string
for i := range genres {
gs := strings.FieldsFunc(genres[i], func(r rune) bool {
return strings.ContainsRune(conf.Server.Scanner.GenreSeparators, r)
})
for j := range gs {
g := strings.TrimSpace(gs[j])
key := strings.ToLower(g)
if _, ok := unique[key]; ok {
continue
}
all = append(all, g)
unique[key] = struct{}{}
}
}
for _, g := range all {
result = append(result, model.Genre{Name: g})
}
if len(result) == 0 {
return "", nil
}
return result[0].Name, result
}
func (s mediaFileMapper) mapDates(md metadata.Tags) (year int, date string,
originalYear int, originalDate string,
releaseYear int, releaseDate string) {
// Start with defaults
year, date = md.Date()
originalYear, originalDate = md.OriginalDate()
releaseYear, releaseDate = md.ReleaseDate()
// MusicBrainz Picard writes the Release Date of an album to the Date tag, and leaves the Release Date tag empty
taggedLikePicard := (originalYear != 0) &&
(releaseYear == 0) &&
(year >= originalYear)
if taggedLikePicard {
return originalYear, originalDate, originalYear, originalDate, year, date
}
// when there's no Date, first fall back to Original Date, then to Release Date.
if year == 0 {
if originalYear > 0 {
year, date = originalYear, originalDate
} else {
year, date = releaseYear, releaseDate
}
}
return year, date, originalYear, originalDate, releaseYear, releaseDate
}

57
scanner2/persist_changes.go Normal file
View File

@ -0,0 +1,57 @@
package scanner2
import (
"context"
"github.com/google/go-pipeline/pkg/pipeline"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/slice"
)
func persistChanges(ctx context.Context) pipeline.StageFn[*folderEntry] {
return func(entry *folderEntry) (*folderEntry, error) {
err := entry.scanCtx.ds.WithTx(func(tx model.DataStore) error {
// Save all tags to DB
err := slice.RangeByChunks(entry.tags, 100, func(chunk []model.Tag) error {
err := tx.Tag(ctx).Add(chunk...)
if err != nil {
log.Error(ctx, "Scanner: Error adding tags to DB", "folder", entry.path, err)
return err
}
return nil
})
if err != nil {
return err
}
// Save all tracks to DB
err = slice.RangeByChunks(entry.tracks, 100, func(chunk []model.MediaFile) error {
for i := range chunk {
track := chunk[i]
err = tx.MediaFile(ctx).Put(&track)
if err != nil {
log.Error(ctx, "Scanner: Error adding/updating mediafile to DB", "folder", entry.path, "track", track, err)
return err
}
}
return nil
})
if err != nil {
return err
}
// Save folder to DB
err = tx.Folder(ctx).Put(entry.scanCtx.lib, entry.path)
if err != nil {
log.Error(ctx, "Scanner: Error adding/updating folder to DB", "folder", entry.path, err)
return err
}
return err
})
if err != nil {
log.Error(ctx, "Scanner: Error persisting changes to DB", "folder", entry.path, err)
}
return entry, err
}
}

1
scanner2/persistent_ids.go Normal file
View File

@ -0,0 +1 @@
package scanner2

96
scanner2/process_folders.go Normal file
View File

@ -0,0 +1,96 @@
package scanner2
import (
"context"
"path/filepath"
"github.com/google/go-pipeline/pkg/pipeline"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/scanner/metadata"
"github.com/navidrome/navidrome/utils/slice"
"golang.org/x/exp/maps"
)
const (
// filesBatchSize used for batching file metadata extraction
filesBatchSize = 100
)
func processFolder(ctx context.Context) pipeline.StageFn[*folderEntry] {
return func(entry *folderEntry) (*folderEntry, error) {
// Load children mediafiles from DB
mfs, err := entry.scanCtx.ds.MediaFile(ctx).GetByFolder(entry.id)
if err != nil {
log.Error(ctx, "Scanner: Error loading mediafiles from DB", "folder", entry.path, err)
return entry, err
}
dbTracks := slice.ToMap(mfs, func(mf model.MediaFile) (string, model.MediaFile) { return mf.Path, mf })
// Get list of files to import, leave in dbTracks only tracks that are missing
var filesToImport []string
for afPath, af := range entry.audioFiles {
fullPath := filepath.Join(entry.path, afPath)
dbTrack, foundInDB := dbTracks[afPath]
if !foundInDB || entry.scanCtx.fullRescan {
filesToImport = append(filesToImport, fullPath)
} else {
info, err := af.Info()
if err != nil {
log.Warn(ctx, "Scanner: Error getting file info", "folder", entry.path, "file", af.Name(), err)
return nil, err
}
if info.ModTime().After(dbTrack.UpdatedAt) {
filesToImport = append(filesToImport, fullPath)
}
}
delete(dbTracks, afPath)
}
// Remaining dbTracks are tracks that were not found in the folder, so they should be marked as missing
entry.missingTracks = maps.Values(dbTracks)
if len(filesToImport) > 0 {
entry.tracks, entry.tags, err = loadTagsFromFiles(ctx, entry, filesToImport)
if err != nil {
log.Warn(ctx, "Scanner: Error loading tags from files. Skipping", "folder", entry.path, err)
return entry, nil
}
entry.albums = loadAlbumsFromTags(ctx, entry)
entry.artists = loadArtistsFromTags(ctx, entry)
}
return entry, nil
}
}
func loadTagsFromFiles(ctx context.Context, entry *folderEntry, toImport []string) (model.MediaFiles, model.FlattenedTags, error) {
tracks := model.MediaFiles{}
uniqueTags := make(map[string]model.Tag)
mapper := newMediaFileMapper(entry)
err := slice.RangeByChunks(toImport, filesBatchSize, func(chunk []string) error {
allFileTags, err := metadata.Extract(toImport...)
if err != nil {
log.Warn(ctx, "Scanner: Error extracting tags from files. Skipping", "folder", entry.path, err)
return err
}
for _, fileTags := range allFileTags {
track := mapper.toMediaFile(fileTags)
tracks = append(tracks, track)
for _, t := range track.Tags.FlattenAll() {
uniqueTags[t.ID] = t
}
}
return nil
})
return tracks, maps.Values(uniqueTags), err
}
func loadAlbumsFromTags(ctx context.Context, entry *folderEntry) model.Albums {
return nil // TODO
}
func loadArtistsFromTags(ctx context.Context, entry *folderEntry) model.Artists {
return nil // TODO
}

231
scanner2/produce_folders.go Normal file
View File

@ -0,0 +1,231 @@
package scanner2
import (
"context"
"io/fs"
"os"
"path/filepath"
"runtime"
"sort"
"strings"
"github.com/google/go-pipeline/pkg/pipeline"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/pl"
"golang.org/x/exp/maps"
)
func produceFolders(ctx context.Context, ds model.DataStore, libs []model.Library, fullRescan bool) pipeline.ProducerFn[*folderEntry] {
scanCtxChan := make(chan *scanContext, len(libs))
go func() {
defer close(scanCtxChan)
for _, lib := range libs {
scanCtx, err := newScannerContext(ctx, ds, lib, fullRescan)
if err != nil {
log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err)
continue
}
scanCtxChan <- scanCtx
}
}()
return func(put func(entry *folderEntry)) error {
// TODO Parallelize multiple scanCtx
var total int64
for scanCtx := range pl.ReadOrDone(ctx, scanCtxChan) {
outputChan, err := walkDirTree(ctx, scanCtx)
if err != nil {
log.Warn(ctx, "Scanner: Error scanning library", "lib", scanCtx.lib.Name, err)
}
for folder := range pl.ReadOrDone(ctx, outputChan) {
put(folder)
}
total += scanCtx.numFolders.Load()
}
log.Info(ctx, "Scanner: Finished loading all folders", "numFolders", total)
return nil
}
}
func walkDirTree(ctx context.Context, scanCtx *scanContext) (<-chan *folderEntry, error) {
results := make(chan *folderEntry)
go func() {
defer close(results)
rootFolder := scanCtx.lib.Path
err := walkFolder(ctx, scanCtx, rootFolder, results)
if err != nil {
log.Error(ctx, "Scanner: There were errors reading directories from filesystem", "path", rootFolder, err)
return
}
log.Debug(ctx, "Scanner: Finished reading folders", "lib", scanCtx.lib.Name, "path", rootFolder, "numFolders", scanCtx.numFolders.Load())
}()
return results, nil
}
func walkFolder(ctx context.Context, scanCtx *scanContext, currentFolder string, results chan<- *folderEntry) error {
folder, children, err := loadDir(ctx, scanCtx, currentFolder)
if err != nil {
log.Warn(ctx, "Scanner: Error loading dir. Skipping", "path", currentFolder, err)
return nil
}
scanCtx.numFolders.Add(1)
for _, c := range children {
err := walkFolder(ctx, scanCtx, c, results)
if err != nil {
return err
}
}
if !folder.isOutdated() && !scanCtx.fullRescan {
return nil
}
dir := filepath.Clean(currentFolder)
log.Trace(ctx, "Scanner: Found directory", "_path", dir, "audioFiles", maps.Keys(folder.audioFiles),
"images", maps.Keys(folder.imageFiles), "playlists", folder.playlists, "imagesUpdatedAt", folder.imagesUpdatedAt,
"updTime", folder.updTime, "modTime", folder.modTime, "numChildren", len(children))
folder.path = dir
results <- folder
return nil
}
func loadDir(ctx context.Context, scanCtx *scanContext, dirPath string) (folder *folderEntry, children []string, err error) {
folder = &folderEntry{scanCtx: scanCtx, path: dirPath}
folder.id = model.FolderID(scanCtx.lib, dirPath)
folder.updTime = scanCtx.getLastUpdatedInDB(folder.id)
folder.audioFiles = make(map[string]fs.DirEntry)
folder.imageFiles = make(map[string]fs.DirEntry)
dirInfo, err := os.Stat(dirPath)
if err != nil {
log.Warn(ctx, "Scanner: Error stating dir", "path", dirPath, err)
return nil, nil, err
}
folder.modTime = dirInfo.ModTime()
dir, err := os.Open(dirPath)
if err != nil {
log.Warn(ctx, "Scanner: Error in Opening directory", "path", dirPath, err)
return folder, children, err
}
defer dir.Close()
for _, entry := range fullReadDir(ctx, dir) {
if ctx.Err() != nil {
return folder, children, ctx.Err()
}
isDir, err := isDirOrSymlinkToDir(dirPath, entry)
// Skip invalid symlinks
if err != nil {
log.Warn(ctx, "Scanner: Invalid symlink", "dir", filepath.Join(dirPath, entry.Name()), err)
continue
}
if isDir && !isDirIgnored(dirPath, entry) && isDirReadable(ctx, dirPath, entry) {
children = append(children, filepath.Join(dirPath, entry.Name()))
} else {
fileInfo, err := entry.Info()
if err != nil {
log.Warn(ctx, "Scanner: Error getting fileInfo", "name", entry.Name(), err)
return folder, children, err
}
if fileInfo.ModTime().After(folder.modTime) {
folder.modTime = fileInfo.ModTime()
}
switch {
case model.IsAudioFile(entry.Name()):
folder.audioFiles[entry.Name()] = entry
case model.IsValidPlaylist(entry.Name()):
folder.playlists = append(folder.playlists, entry)
case model.IsImageFile(entry.Name()):
folder.imageFiles[entry.Name()] = entry
if fileInfo.ModTime().After(folder.imagesUpdatedAt) {
folder.imagesUpdatedAt = fileInfo.ModTime()
}
}
}
}
return folder, children, nil
}
// fullReadDir reads all files in the folder, skipping the ones with errors.
// It also detects when it is "stuck" with an error in the same directory over and over.
// In this case, it stops and returns whatever it was able to read until it got stuck.
// See discussion here: https://github.com/navidrome/navidrome/issues/1164#issuecomment-881922850
func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []fs.DirEntry {
var allEntries []fs.DirEntry
var prevErrStr = ""
for {
if ctx.Err() != nil {
return []fs.DirEntry{}
}
entries, err := dir.ReadDir(-1)
allEntries = append(allEntries, entries...)
if err == nil {
break
}
log.Warn(ctx, "Skipping DirEntry", err)
if prevErrStr == err.Error() {
log.Error(ctx, "Scanner: Duplicate DirEntry failure, bailing", err)
break
}
prevErrStr = err.Error()
}
sort.Slice(allEntries, func(i, j int) bool { return allEntries[i].Name() < allEntries[j].Name() })
return allEntries
}
// isDirOrSymlinkToDir returns true if and only if the dirEnt represents a file
// system directory, or a symbolic link to a directory. Note that if the dirEnt
// is not a directory but is a symbolic link, this method will resolve by
// sending a request to the operating system to follow the symbolic link.
// originally copied from github.com/karrick/godirwalk, modified to use dirEntry for
// efficiency for go 1.16 and beyond
func isDirOrSymlinkToDir(baseDir string, dirEnt fs.DirEntry) (bool, error) {
if dirEnt.IsDir() {
return true, nil
}
if dirEnt.Type()&os.ModeSymlink == 0 {
return false, nil
}
// Does this symlink point to a directory?
fileInfo, err := os.Stat(filepath.Join(baseDir, dirEnt.Name()))
if err != nil {
return false, err
}
return fileInfo.IsDir(), nil
}
// isDirReadable returns true if the directory represented by dirEnt is readable
func isDirReadable(ctx context.Context, baseDir string, dirEnt fs.DirEntry) bool {
path := filepath.Join(baseDir, dirEnt.Name())
dir, err := os.Open(path)
if err != nil {
log.Warn("Scanner: Skipping unreadable directory", "path", path, err)
return false
}
err = dir.Close()
if err != nil {
log.Warn(ctx, "Scanner: Error closing directory", "path", path, err)
}
return true
}
// isDirIgnored returns true if the directory represented by dirEnt contains an
// `ignore` file (named after skipScanFile)
func isDirIgnored(baseDir string, dirEnt fs.DirEntry) bool {
// allows Album folders for albums which eg start with ellipses
name := dirEnt.Name()
if strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..") {
return true
}
if runtime.GOOS == "windows" && strings.EqualFold(name, "$RECYCLE.BIN") {
return true
}
_, err := os.Stat(filepath.Join(baseDir, name, consts.SkipScanFile))
return err == nil
}

46
scanner2/scan_context.go Normal file
View File

@ -0,0 +1,46 @@
package scanner2
import (
"context"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/navidrome/navidrome/model"
)
type scanContext struct {
lib model.Library
ds model.DataStore
startTime time.Time
lastUpdates map[string]time.Time
lock sync.RWMutex
fullRescan bool
numFolders atomic.Int64
}
func newScannerContext(ctx context.Context, ds model.DataStore, lib model.Library, fullRescan bool) (*scanContext, error) {
lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib)
if err != nil {
return nil, fmt.Errorf("error getting last updates: %w", err)
}
return &scanContext{
lib: lib,
ds: ds,
startTime: time.Now(),
lastUpdates: lastUpdates,
fullRescan: fullRescan,
}, nil
}
func (s *scanContext) getLastUpdatedInDB(id string) time.Time {
s.lock.RLock()
defer s.lock.RUnlock()
t, ok := s.lastUpdates[id]
if !ok {
return time.Time{}
}
return t
}

75
scanner2/scanner2.go Normal file
View File

@ -0,0 +1,75 @@
package scanner2
import (
"context"
"time"
"github.com/google/go-pipeline/pkg/pipeline"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/scanner"
)
type scanner2 struct {
processCtx context.Context
ds model.DataStore
}
func New(ctx context.Context, ds model.DataStore) scanner.Scanner {
return &scanner2{processCtx: ctx, ds: ds}
}
func (s *scanner2) RescanAll(requestCtx context.Context, fullRescan bool) error {
ctx := request.AddValues(s.processCtx, requestCtx)
libs, err := s.ds.Library(ctx).GetAll()
if err != nil {
return err
}
startTime := time.Now()
log.Info(ctx, "Scanner: Starting scan", "fullRescan", fullRescan, "numLibraries", len(libs))
err = s.runPipeline(
pipeline.NewProducer(produceFolders(ctx, s.ds, libs, fullRescan), pipeline.Name("read folders from disk")),
pipeline.NewStage(processFolder(ctx), pipeline.Name("process folder")),
pipeline.NewStage(persistChanges(ctx), pipeline.Name("persist changes")),
pipeline.NewStage(logFolder(ctx), pipeline.Name("log results")),
)
if err != nil {
log.Error(ctx, "Scanner: Error scanning libraries", "duration", time.Since(startTime), err)
} else {
log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime))
}
return err
}
func (s *scanner2) runPipeline(producer pipeline.Producer[*folderEntry], stages ...pipeline.Stage[*folderEntry]) error {
if log.IsGreaterOrEqualTo(log.LevelDebug) {
metrics, err := pipeline.Measure(producer, stages...)
log.Info(metrics.String(), err)
return err
}
return pipeline.Do(producer, stages...)
}
func logFolder(ctx context.Context) func(folder *folderEntry) (out *folderEntry, err error) {
return func(folder *folderEntry) (out *folderEntry, err error) {
log.Debug(ctx, "Scanner: Completed processing folder", "_path", folder.path,
"audioCount", len(folder.audioFiles), "imageCount", len(folder.imageFiles), "plsCount", len(folder.playlists))
return folder, nil
}
}
func (s *scanner2) Status(context.Context) (*scanner.StatusInfo, error) {
return &scanner.StatusInfo{}, nil
}
//nolint:unused
func (s *scanner2) doScan(ctx context.Context, fullRescan bool, folders <-chan string) error {
return nil
}
var _ scanner.Scanner = (*scanner2)(nil)

7
server/initial_setup.go
View File

@ -15,8 +15,13 @@ import (
)
func initialSetup(ds model.DataStore) {
ctx := context.TODO()
_ = ds.WithTx(func(tx model.DataStore) error {
properties := ds.Property(context.TODO())
if err := ds.Library(ctx).StoreMusicFolder(); err != nil {
return err
}
properties := ds.Property(ctx)
_, err := properties.Get(consts.InitialSetupFlagKey)
if err == nil {
return nil

27
server/subsonic/browsing.go
View File

@ -4,7 +4,6 @@ import (
"context"
"errors"
"net/http"
"strconv"
"time"
"github.com/navidrome/navidrome/conf"
@ -18,10 +17,10 @@ import (
)
func (api *Router) GetMusicFolders(r *http.Request) (*responses.Subsonic, error) {
mediaFolderList, _ := api.ds.MediaFolder(r.Context()).GetAll()
folders := make([]responses.MusicFolder, len(mediaFolderList))
for i, f := range mediaFolderList {
folders[i].Id = f.ID
libraries, _ := api.ds.Library(r.Context()).GetAll()
folders := make([]responses.MusicFolder, len(libraries))
for i, f := range libraries {
folders[i].Id = int32(f.ID)
folders[i].Name = f.Name
}
response := newResponse()
@ -29,24 +28,16 @@ func (api *Router) GetMusicFolders(r *http.Request) (*responses.Subsonic, error)
return response, nil
}
func (api *Router) getArtistIndex(r *http.Request, mediaFolderId int, ifModifiedSince time.Time) (*responses.Indexes, error) {
func (api *Router) getArtistIndex(r *http.Request, libId int, ifModifiedSince time.Time) (*responses.Indexes, error) {
ctx := r.Context()
folder, err := api.ds.MediaFolder(ctx).Get(int32(mediaFolderId))
lib, err := api.ds.Library(ctx).Get(libId)
if err != nil {
log.Error(ctx, "Error retrieving MediaFolder", "id", mediaFolderId, err)
return nil, err
}
l, err := api.ds.Property(ctx).DefaultGet(model.PropLastScan+"-"+folder.Path, "-1")
if err != nil {
log.Error(ctx, "Error retrieving LastScan property", err)
log.Error(ctx, "Error retrieving Library", "id", libId, err)
return nil, err
}
var indexes model.ArtistIndexes
ms, _ := strconv.ParseInt(l, 10, 64)
lastModified := utils.ToTime(ms)
if lastModified.After(ifModifiedSince) {
if lib.LastScanAt.After(ifModifiedSince) {
indexes, err = api.ds.Artist(ctx).GetIndex()
if err != nil {
log.Error(ctx, "Error retrieving Indexes", err)
@ -56,7 +47,7 @@ func (api *Router) getArtistIndex(r *http.Request, mediaFolderId int, ifModified
res := &responses.Indexes{
IgnoredArticles: conf.Server.IgnoredArticles,
LastModified: utils.ToMillis(lastModified),
LastModified: utils.ToMillis(lib.LastScanAt),
}
res.Index = make([]responses.Index, len(indexes))

5
server/subsonic/library_scanning.go
View File

@ -4,7 +4,6 @@ import (
"net/http"
"time"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/server/subsonic/responses"
@ -12,10 +11,8 @@ import (
)
func (api *Router) GetScanStatus(r *http.Request) (*responses.Subsonic, error) {
// TODO handle multiple mediafolders
ctx := r.Context()
mediaFolder := conf.Server.MusicFolder
status, err := api.scanner.Status(mediaFolder)
status, err := api.scanner.Status(ctx)
if err != nil {
log.Error(ctx, "Error retrieving Scanner status", err)
return nil, newError(responses.ErrorGeneric, "Internal Error")

12
tests/mock_persistence.go
View File

@ -43,8 +43,16 @@ func (db *MockDataStore) MediaFile(context.Context) model.MediaFileRepository {
return db.MockedMediaFile
}
func (db *MockDataStore) MediaFolder(context.Context) model.MediaFolderRepository {
return struct{ model.MediaFolderRepository }{}
func (db *MockDataStore) Library(context.Context) model.LibraryRepository {
return struct{ model.LibraryRepository }{}
}
func (db *MockDataStore) Folder(context.Context) model.FolderRepository {
return struct{ model.FolderRepository }{}
}
func (db *MockDataStore) Tag(context.Context) model.TagRepository {
return struct{ model.TagRepository }{}
}
func (db *MockDataStore) Genre(context.Context) model.GenreRepository {

29
utils/pl/pipelines.go
View File

@ -174,3 +174,32 @@ func FromSlice[T any](ctx context.Context, in []T) <-chan T {
close(output)
return output
}
func Filter[T any](ctx context.Context, maxWorkers int, inputChan chan T, f func(context.Context, T) (bool, error)) (chan T, chan error) {
outputChan := make(chan T)
errorChan := make(chan error)
go func() {
defer close(outputChan)
defer close(errorChan)
errChan := Sink(ctx, maxWorkers, inputChan, func(ctx context.Context, item T) error {
ok, err := f(ctx, item)
if err != nil {
return err
}
if ok {
outputChan <- item
}
return nil
})
// Wait for pipeline to end, and forward any errors
for err := range ReadOrDone(ctx, errChan) {
select {
case errorChan <- err:
default:
}
}
}()
return outputChan, errorChan
}

9
utils/slice/slice.go
View File

@ -17,6 +17,15 @@ func Group[T any, K comparable](s []T, keyFunc func(T) K) map[K][]T {
return m
}
func ToMap[T any, K comparable, V any](s []T, transformFunc func(T) (K, V)) map[K]V {
m := map[K]V{}
for _, item := range s {
k, v := transformFunc(item)
m[k] = v
}
return m
}
func MostFrequent[T comparable](list []T) T {
if len(list) == 0 {
var zero T

18
utils/slice/slice_test.go
View File

@ -45,6 +45,24 @@ var _ = Describe("Slice Utils", func() {
})
})
Describe("ToMap", func() {
It("returns empty map for an empty input", func() {
transformFunc := func(v int) (int, string) { return v, strconv.Itoa(v) }
result := slice.ToMap([]int{}, transformFunc)
Expect(result).To(BeEmpty())
})
It("returns a map with the result of the transform function", func() {
transformFunc := func(v int) (int, string) { return v * 2, strconv.Itoa(v * 2) }
result := slice.ToMap([]int{1, 2, 3, 4}, transformFunc)
Expect(result).To(HaveLen(4))
Expect(result).To(HaveKeyWithValue(2, "2"))
Expect(result).To(HaveKeyWithValue(4, "4"))
Expect(result).To(HaveKeyWithValue(6, "6"))
Expect(result).To(HaveKeyWithValue(8, "8"))
})
})
Describe("MostFrequent", func() {
It("returns zero value if no arguments are passed", func() {
Expect(slice.MostFrequent([]int{})).To(BeZero())