mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

New Folder Scanner - WIP

This commit is contained in:
Deluan 2020-01-16 16:53:48 -05:00
parent 7a16d41abe
commit 123f543a94
27 changed files with 1092 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -10,4 +10,4 @@ Artwork
sonic.toml
master.zip
Jamstash-master
storm.db
testDB

2
Makefile
View File

@ -21,7 +21,7 @@ test: check_go_env
.PHONY: build
build: check_go_env
go build
@(cd ./ui && npm run build)
# @(cd ./ui && npm run build)
.PHONY: setup
setup: Jamstash-master

2
conf/configuration.go
View File

@ -20,6 +20,7 @@ type sonic struct {
DisableDownsampling bool `default:"false"`
DownsampleCommand string `default:"ffmpeg -i %s -map 0:0 -b:a %bk -v 0 -f mp3 -"`
ProbeCommand string `default:"ffprobe -v quiet -print_format json -show_format %s"`
PlsIgnoreFolders bool `default:"true"`
PlsIgnoredPatterns string `default:"^iCloud;\\~"`
@ -28,6 +29,7 @@ type sonic struct {
DevDisableAuthentication bool `default:"false"`
DevDisableFileCheck bool `default:"false"`
DevDisableBanner bool `default:"false"`
DevUseFileScanner bool `default:"false"`
}
var Sonic *sonic

2
go.mod
View File

@ -7,7 +7,6 @@ require (
github.com/Masterminds/squirrel v1.1.0
github.com/astaxie/beego v1.12.0
github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
github.com/deluan/rest v0.0.0-20200114062534-0653ffe9eab4
github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131
github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a // indirect
github.com/dhowden/tag v0.0.0-20191122115059-7e5c04feccd8
@ -15,7 +14,6 @@ require (
github.com/fatih/structs v1.0.0 // indirect
github.com/go-chi/chi v4.0.3+incompatible
github.com/go-chi/cors v1.0.0
github.com/google/uuid v1.1.1
github.com/google/wire v0.4.0
github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629
github.com/koding/multiconfig v0.0.0-20170327155832-26b6dfd3a84a

8
go.sum
View File

@ -22,14 +22,10 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deluan/rest v0.0.0-20200114062534-0653ffe9eab4 h1:VSoAcWJvj656TSyWbJ5KuGsi/J8dO5+iO9+5/7I8wao=
github.com/deluan/rest v0.0.0-20200114062534-0653ffe9eab4/go.mod h1:tSgDythFsl0QgS/PFWfIZqcJKnkADWneY80jaVRlqK8=
github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131 h1:siEGb+iB1Ea75U7BnkYVSqSRzE6QHlXCbqEXenxRmhQ=
github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131/go.mod h1:eVWQJVQ67aMvYhpkDwaH2Goy2vo6v8JCMfGXfQ9sPtw=
github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a h1:7MucP9rMAsQRcRE1sGpvMZoTxFYZlDmfDvCH+z7H+90=
github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a/go.mod h1:sLjdR6uwx3L6/Py8F+QgAfeiuY87xuYGwCDqRFrvCzw=
github.com/dhowden/tag v0.0.0-20170128231422-9edd38ca5d10 h1:H4waV2XHwsud7v90IfEbwwynwVlC7YU3RicLlfkfgn4=
github.com/dhowden/tag v0.0.0-20170128231422-9edd38ca5d10/go.mod h1:SniNVYuaD1jmdEEvi+7ywb1QFR7agjeTdGKyFb0p7Rw=
github.com/dhowden/tag v0.0.0-20191122115059-7e5c04feccd8 h1:nmFnmD8VZXkjDHIb1Gnfz50cgzUvGN72zLjPRXBW/hU=
github.com/dhowden/tag v0.0.0-20191122115059-7e5c04feccd8/go.mod h1:SniNVYuaD1jmdEEvi+7ywb1QFR7agjeTdGKyFb0p7Rw=
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8=
@ -58,8 +54,6 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE=
github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@ -84,8 +78,6 @@ github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtB
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=

2
log/log.go
View File

@ -132,7 +132,7 @@ func parseArgs(args []interface{}) (*logrus.Entry, string) {
kvPairs := args[1:]
l = addFields(l, kvPairs)
}
if currentLevel >= LevelDebug {
if currentLevel >= LevelTrace {
_, file, line, ok := runtime.Caller(2)
if !ok {
file = "???"

1
model/album.go
View File

@ -37,4 +37,5 @@ type AlbumRepository interface {
GetAllIds() ([]string, error)
GetStarred(...QueryOptions) (Albums, error)
Search(q string, offset int, size int) (Albums, error)
Refresh(ids ...string) error
}

1
model/artist.go
View File

@ -13,6 +13,7 @@ type ArtistRepository interface {
Get(id string) (*Artist, error)
PurgeInactive(active Artists) error
Search(q string, offset int, size int) (Artists, error)
Refresh(ids ...string) error
}
type Artists []Artist

5
model/index.go
View File

@ -14,11 +14,10 @@ type ArtistIndex struct {
type ArtistInfos []ArtistInfo
type ArtistIndexes []ArtistIndex
// TODO Combine ArtistIndex with Artist
type ArtistIndexRepository interface {
CountAll() (int64, error)
Exists(id string) (bool, error)
Put(m *ArtistIndex) error
Get(id string) (*ArtistIndex, error)
Refresh() error
GetAll() (ArtistIndexes, error)
DeleteAll() error
}

1
model/mediafile.go
View File

@ -51,4 +51,5 @@ type MediaFileRepository interface {
GetAllIds() ([]string, error)
Search(q string, offset int, size int) (MediaFiles, error)
Delete(id string) error
DeleteByPath(path string) error
}

60
persistence/album_repository.go
View File

@ -1,9 +1,12 @@
package persistence
import (
"fmt"
"strings"
"time"
"github.com/astaxie/beego/orm"
"github.com/cloudsonic/sonic-server/log"
"github.com/cloudsonic/sonic-server/model"
)
@ -85,7 +88,62 @@ func (r *albumRepository) toAlbums(all []Album) model.Albums {
return result
}
// TODO Remove []string from return
func (r *albumRepository) Refresh(ids ...string) error {
type refreshAlbum struct {
Album
CurrentId string
HasCoverArt bool
}
var albums []refreshAlbum
o := Db()
sql := fmt.Sprintf(`
select album_id as id, album as name, f.artist, f.album_artist, f.artist_id, f.compilation, f.genre,
max(f.year) as year, sum(f.play_count) as play_count, max(f.play_date) as play_date, sum(f.duration) as duration,
max(f.updated_at) as updated_at, min(f.created_at) as created_at, count(*) as song_count,
a.id as current_id, f.id as cover_art_id, f.path as cover_art_path, f.has_cover_art
from media_file f left outer join album a on f.album_id = a.id
where f.album_id in ('%s')
group by album_id order by f.id`, strings.Join(ids, "','"))
_, err := o.Raw(sql).QueryRows(&albums)
if err != nil {
return err
}
var toInsert []Album
var toUpdate []Album
for _, al := range albums {
if !al.HasCoverArt {
al.CoverArtId = ""
}
if al.Compilation {
al.AlbumArtist = "Various Artists"
}
if al.CurrentId != "" {
toUpdate = append(toUpdate, al.Album)
} else {
toInsert = append(toInsert, al.Album)
}
}
if len(toInsert) > 0 {
n, err := o.InsertMulti(100, toInsert)
if err != nil {
return err
}
log.Debug("Inserted new albums", "num", n)
}
if len(toUpdate) > 0 {
for _, al := range toUpdate {
_, err := o.Update(&al, "name", "artist_id", "cover_art_path", "cover_art_id", "artist", "album_artist", "year",
"compilation", "play_count", "song_count", "duration", "updated_at")
if err != nil {
return err
}
}
log.Debug("Updated albums", "num", len(toUpdate))
}
return err
}
func (r *albumRepository) PurgeInactive(activeList model.Albums) error {
return withTx(func(o orm.Ormer) error {
_, err := r.purgeInactive(o, activeList, func(item interface{}) string {

62
persistence/artist_repository.go
View File

@ -1,7 +1,11 @@
package persistence
import (
"fmt"
"strings"
"github.com/astaxie/beego/orm"
"github.com/cloudsonic/sonic-server/log"
"github.com/cloudsonic/sonic-server/model"
)
@ -42,6 +46,64 @@ func (r *artistRepository) Get(id string) (*model.Artist, error) {
return &a, nil
}
func (r *artistRepository) Refresh(ids ...string) error {
type refreshArtist struct {
Artist
CurrentId string
AlbumArtist string
Compilation bool
}
var artists []refreshArtist
o := Db()
sql := fmt.Sprintf(`
select f.artist_id as id,
f.artist as name,
f.album_artist,
f.compilation,
count(*) as album_count,
a.id as current_id
from album f
left outer join artist a on f.artist_id = a.id
where f.artist_id in ('%s') group by f.artist_id order by f.id`, strings.Join(ids, "','"))
_, err := o.Raw(sql).QueryRows(&artists)
if err != nil {
return err
}
var toInsert []Artist
var toUpdate []Artist
for _, al := range artists {
if al.Compilation {
al.AlbumArtist = "Various Artists"
}
if al.AlbumArtist != "" {
al.Name = al.AlbumArtist
}
if al.CurrentId != "" {
toUpdate = append(toUpdate, al.Artist)
} else {
toInsert = append(toInsert, al.Artist)
}
}
if len(toInsert) > 0 {
n, err := o.InsertMulti(100, toInsert)
if err != nil {
return err
}
log.Debug("Inserted new artists", "num", n)
}
if len(toUpdate) > 0 {
for _, al := range toUpdate {
_, err := o.Update(&al, "name", "album_count")
if err != nil {
return err
}
}
log.Debug("Updated artists", "num", len(toUpdate))
}
return err
}
func (r *artistRepository) PurgeInactive(activeList model.Artists) error {
return withTx(func(o orm.Ormer) error {
_, err := r.purgeInactive(o, activeList, func(item interface{}) string {

79
persistence/index_repository.go
View File

@ -2,9 +2,12 @@ package persistence
import (
"sort"
"strings"
"github.com/astaxie/beego/orm"
"github.com/cloudsonic/sonic-server/conf"
"github.com/cloudsonic/sonic-server/model"
"github.com/cloudsonic/sonic-server/utils"
)
type ArtistInfo struct {
@ -15,6 +18,8 @@ type ArtistInfo struct {
AlbumCount int
}
type tempIndex map[string]model.ArtistInfo
type artistIndexRepository struct {
sqlRepository
}
@ -25,15 +30,6 @@ func NewArtistIndexRepository() model.ArtistIndexRepository {
return r
}
func (r *artistIndexRepository) CountAll() (int64, error) {
count := struct{ Count int64 }{}
err := Db().Raw("select count(distinct(idx)) as count from artist_info").QueryRow(&count)
if err != nil {
return 0, err
}
return count.Count, nil
}
func (r *artistIndexRepository) Put(idx *model.ArtistIndex) error {
return withTx(func(o orm.Ormer) error {
_, err := r.newQuery(o).Filter("idx", idx.ID).Delete()
@ -56,23 +52,64 @@ func (r *artistIndexRepository) Put(idx *model.ArtistIndex) error {
})
}
func (r *artistIndexRepository) Get(id string) (*model.ArtistIndex, error) {
var ais []ArtistInfo
_, err := r.newQuery(Db()).Filter("idx", id).All(&ais)
func (r *artistIndexRepository) Refresh() error {
o := Db()
indexGroups := utils.ParseIndexGroups(conf.Sonic.IndexGroups)
artistIndex := make(map[string]tempIndex)
var artists []Artist
_, err := o.QueryTable(&Artist{}).All(&artists)
if err != nil {
return nil, err
return err
}
idx := &model.ArtistIndex{ID: id}
idx.Artists = make([]model.ArtistInfo, len(ais))
for i, a := range ais {
idx.Artists[i] = model.ArtistInfo{
ArtistID: a.ArtistID,
Artist: a.Artist,
AlbumCount: a.AlbumCount,
for _, ar := range artists {
r.collectIndex(indexGroups, &ar, artistIndex)
}
return r.saveIndex(artistIndex)
}
func (r *artistIndexRepository) collectIndex(ig utils.IndexGroups, a *Artist, artistIndex map[string]tempIndex) {
name := a.Name
indexName := strings.ToLower(utils.NoArticle(name))
if indexName == "" {
return
}
group := r.findGroup(ig, indexName)
artists := artistIndex[group]
if artists == nil {
artists = make(tempIndex)
artistIndex[group] = artists
}
artists[indexName] = model.ArtistInfo{ArtistID: a.ID, Artist: a.Name, AlbumCount: a.AlbumCount}
}
func (r *artistIndexRepository) findGroup(ig utils.IndexGroups, name string) string {
for k, v := range ig {
key := strings.ToLower(k)
if strings.HasPrefix(name, key) {
return v
}
}
return idx, err
return "#"
}
func (r *artistIndexRepository) saveIndex(artistIndex map[string]tempIndex) error {
r.DeleteAll()
for k, temp := range artistIndex {
idx := &model.ArtistIndex{ID: k}
for _, v := range temp {
idx.Artists = append(idx.Artists, v)
}
err := r.Put(idx)
if err != nil {
return err
}
}
return nil
}
func (r *artistIndexRepository) GetAll() (model.ArtistIndexes, error) {

5
persistence/index_repository_test.go
View File

@ -35,11 +35,6 @@ var _ = Describe("Artist Index", func() {
Expect(repo.Put(&idx1)).To(BeNil())
Expect(repo.Put(&idx2)).To(BeNil())
Expect(repo.Get("D")).To(Equal(&idx1))
Expect(repo.Get("S")).To(Equal(&idx2))
Expect(repo.GetAll()).To(Equal(model.ArtistIndexes{idx1, idx2}))
Expect(repo.CountAll()).To(Equal(int64(2)))
Expect(repo.DeleteAll()).To(BeNil())
Expect(repo.CountAll()).To(Equal(int64(0)))
})
})

20
persistence/mediafile_repository.go
View File

@ -102,6 +102,26 @@ func (r *mediaFileRepository) FindByPath(path string) (model.MediaFiles, error)
return r.toMediaFiles(filtered), nil
}
func (r *mediaFileRepository) DeleteByPath(path string) error {
o := Db()
var mfs []MediaFile
_, err := r.newQuery(o).Filter("path__istartswith", path).OrderBy("disc_number", "track_number").All(&mfs)
if err != nil {
return err
}
var filtered []string
path = strings.ToLower(path) + string(os.PathSeparator)
for _, mf := range mfs {
filename := strings.TrimPrefix(strings.ToLower(mf.Path), path)
if len(strings.Split(filename, string(os.PathSeparator))) > 1 {
continue
}
filtered = append(filtered, mf.ID)
}
_, err = r.newQuery(o).Filter("id__in", filtered).Delete()
return err
}
func (r *mediaFileRepository) GetStarred(options ...model.QueryOptions) (model.MediaFiles, error) {
var starred []MediaFile
_, err := r.newQuery(Db(), options...).Filter("starred", true).All(&starred)

113
scanner/change_detector.go Normal file
View File

@ -0,0 +1,113 @@
package scanner
import (
"os"
"path"
"strings"
"time"
"github.com/cloudsonic/sonic-server/log"
)
type ChangeDetector struct {
rootFolder string
lastUpdate time.Time
dirMap map[string]time.Time
}
func NewChangeDetector(rootFolder string, lastUpdate time.Time) *ChangeDetector {
return &ChangeDetector{
rootFolder: rootFolder,
lastUpdate: lastUpdate,
dirMap: map[string]time.Time{},
}
}
func (s *ChangeDetector) Scan() (changed []string, deleted []string, err error) {
start := time.Now()
newMap := make(map[string]time.Time)
err = s.loadMap(s.rootFolder, newMap)
if err != nil {
return
}
changed, deleted, err = s.checkForUpdates(s.dirMap, newMap)
if err != nil {
return
}
elapsed := time.Since(start)
log.Trace("Folder analysis complete\n", "total", len(newMap), "changed", len(changed), "deleted", len(deleted), "elapsed", elapsed)
s.dirMap = newMap
return
}
func (s *ChangeDetector) loadDir(dirPath string) (children []string, lastUpdated time.Time, err error) {
dir, err := os.Open(dirPath)
if err != nil {
return
}
dirInfo, err := os.Stat(dirPath)
if err != nil {
return
}
lastUpdated = dirInfo.ModTime()
files, err := dir.Readdir(-1)
if err != nil {
return
}
for _, f := range files {
if f.IsDir() {
children = append(children, path.Join(dirPath, f.Name()))
} else {
if f.ModTime().After(lastUpdated) {
lastUpdated = f.ModTime()
}
}
}
return
}
func (s *ChangeDetector) loadMap(rootPath string, dirMap map[string]time.Time) error {
children, lastUpdated, err := s.loadDir(rootPath)
if err != nil {
return err
}
for _, c := range children {
err := s.loadMap(c, dirMap)
if err != nil {
return err
}
}
dir := s.getRelativePath(rootPath)
dirMap[dir] = lastUpdated
return nil
}
func (s *ChangeDetector) getRelativePath(subfolder string) string {
dir := strings.TrimPrefix(subfolder, s.rootFolder)
if dir == "" {
dir = "."
}
return dir
}
func (s *ChangeDetector) checkForUpdates(oldMap map[string]time.Time, newMap map[string]time.Time) (changed []string, deleted []string, err error) {
for dir, lastUpdated := range newMap {
oldLastUpdated, ok := oldMap[dir]
if !ok {
oldLastUpdated = s.lastUpdate
}
if lastUpdated.After(oldLastUpdated) {
changed = append(changed, dir)
}
}
for dir := range oldMap {
if _, ok := newMap[dir]; !ok {
deleted = append(deleted, dir)
}
}
return
}

103
scanner/change_detector_test.go Normal file
View File

@ -0,0 +1,103 @@
package scanner
import (
"io/ioutil"
"os"
"path"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("ChangeDetector", func() {
var testFolder string
var scanner *ChangeDetector
BeforeEach(func() {
testFolder, _ = ioutil.TempDir("", "cloudsonic_tests")
err := os.MkdirAll(testFolder, 0700)
if err != nil {
panic(err)
}
scanner = NewChangeDetector(testFolder, time.Time{})
})
It("detects changes recursively", func() {
// Scan empty folder
changed, deleted, err := scanner.Scan()
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf("."))
// Add one subfolder
err = os.MkdirAll(path.Join(testFolder, "a"), 0700)
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan()
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf(".", "/a"))
// Add more subfolders
err = os.MkdirAll(path.Join(testFolder, "a", "b", "c"), 0700)
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan()
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf("/a", "/a/b", "/a/b/c"))
// Scan with no changes
changed, deleted, err = scanner.Scan()
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(BeEmpty())
// New file in subfolder
_, err = os.Create(path.Join(testFolder, "a", "b", "empty.txt"))
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan()
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf("/a/b"))
// Delete file in subfolder
err = os.Remove(path.Join(testFolder, "a", "b", "empty.txt"))
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan()
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf("/a/b"))
// Delete subfolder
err = os.Remove(path.Join(testFolder, "a", "b", "c"))
if err != nil {
panic(err)
}
changed, deleted, err = scanner.Scan()
Expect(err).To(BeNil())
Expect(deleted).To(ConsistOf("/a/b/c"))
Expect(changed).To(ConsistOf("/a/b"))
// Only returns changes after lastUpdate
newScanner := NewChangeDetector(testFolder, time.Now())
changed, deleted, err = newScanner.Scan()
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(BeEmpty())
Expect(changed).To(BeEmpty())
_, err = os.Create(path.Join(testFolder, "a", "b", "new.txt"))
changed, deleted, err = newScanner.Scan()
Expect(err).To(BeNil())
Expect(deleted).To(BeEmpty())
Expect(changed).To(ConsistOf("/a/b"))
})
})

140
scanner/metadata.go Normal file
View File

@ -0,0 +1,140 @@
package scanner
import (
"encoding/json"
"errors"
"os"
"os/exec"
"path"
"strconv"
"strings"
"time"
"github.com/cloudsonic/sonic-server/conf"
"github.com/cloudsonic/sonic-server/log"
"github.com/dhowden/tag"
)
type Metadata struct {
filePath string
suffix string
fileInfo os.FileInfo
t tag.Metadata
duration int
bitRate int
compilation bool
}
func ExtractMetadata(filePath string) (*Metadata, error) {
m := &Metadata{filePath: filePath}
m.suffix = strings.ToLower(strings.TrimPrefix(path.Ext(filePath), "."))
fi, err := os.Stat(filePath)
if err != nil {
return nil, err
}
m.fileInfo = fi
f, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer f.Close()
t, err := tag.ReadFrom(f)
if err != nil {
return nil, err
}
m.t = t
err = m.probe(filePath)
return m, err
}
func (m *Metadata) Title() string { return m.t.Title() }
func (m *Metadata) Album() string { return m.t.Album() }
func (m *Metadata) Artist() string { return m.t.Artist() }
func (m *Metadata) AlbumArtist() string { return m.t.AlbumArtist() }
func (m *Metadata) Composer() string { return m.t.Composer() }
func (m *Metadata) Genre() string { return m.t.Genre() }
func (m *Metadata) Year() int { return m.t.Year() }
func (m *Metadata) TrackNumber() (int, int) { return m.t.Track() }
func (m *Metadata) DiscNumber() (int, int) { return m.t.Disc() }
func (m *Metadata) HasPicture() bool { return m.t.Picture() != nil }
func (m *Metadata) Compilation() bool { return m.compilation }
func (m *Metadata) Duration() int { return m.duration }
func (m *Metadata) BitRate() int { return m.bitRate }
func (m *Metadata) ModificationTime() time.Time { return m.fileInfo.ModTime() }
func (m *Metadata) FilePath() string { return m.filePath }
func (m *Metadata) Suffix() string { return m.suffix }
func (m *Metadata) Size() int { return int(m.fileInfo.Size()) }
// probe analyzes the file and returns duration in seconds and bitRate in kb/s.
// It uses the ffprobe external tool, configured in conf.Sonic.ProbeCommand
func (m *Metadata) probe(filePath string) error {
cmdLine, args := createProbeCommand(filePath)
log.Trace("Executing command", "cmdLine", cmdLine, "args", args)
cmd := exec.Command(cmdLine, args...)
output, err := cmd.CombinedOutput()
if err != nil {
return err
}
return m.parseOutput(output)
}
func (m *Metadata) parseInt(objItf interface{}, field string) (int, error) {
obj := objItf.(map[string]interface{})
s, ok := obj[field].(string)
if !ok {
return -1, errors.New("invalid ffprobe output field obj." + field)
}
fDuration, err := strconv.ParseFloat(s, 64)
if err != nil {
return -1, err
}
return int(fDuration), nil
}
func (m *Metadata) parseOutput(output []byte) error {
var data map[string]map[string]interface{}
err := json.Unmarshal(output, &data)
if err != nil {
return err
}
format, ok := data["format"]
if !ok {
err = errors.New("invalid ffprobe output. no format found")
return err
}
if tags, ok := format["tags"]; ok {
c, _ := m.parseInt(tags, "compilation")
m.compilation = c == 1
}
m.duration, err = m.parseInt(format, "duration")
if err != nil {
return err
}
m.bitRate, err = m.parseInt(format, "bit_rate")
m.bitRate = m.bitRate / 1000
if err != nil {
return err
}
return nil
}
func createProbeCommand(filePath string) (string, []string) {
cmd := conf.Sonic.ProbeCommand
split := strings.Split(cmd, " ")
for i, s := range split {
s = strings.Replace(s, "%s", filePath, -1)
split[i] = s
}
return split[0], split[1:]
}

55
scanner/metadata_test.go Normal file
View File

@ -0,0 +1,55 @@
package scanner
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Metadata", func() {
It("correctly parses mp3 file", func() {
m, err := ExtractMetadata("../tests/fixtures/test.mp3")
Expect(err).To(BeNil())
Expect(m.Title()).To(Equal("Song"))
Expect(m.Album()).To(Equal("Album"))
Expect(m.Artist()).To(Equal("Artist"))
Expect(m.AlbumArtist()).To(Equal("Album Artist"))
Expect(m.Composer()).To(Equal("Composer"))
Expect(m.Compilation()).To(BeFalse())
Expect(m.Genre()).To(Equal("Rock"))
Expect(m.Year()).To(Equal(2014))
n, t := m.TrackNumber()
Expect(n).To(Equal(2))
Expect(t).To(Equal(10))
n, t = m.DiscNumber()
Expect(n).To(Equal(1))
Expect(t).To(Equal(2))
Expect(m.HasPicture()).To(BeTrue())
Expect(m.Duration()).To(Equal(1))
Expect(m.BitRate()).To(Equal(476))
Expect(m.FilePath()).To(Equal("../tests/fixtures/test.mp3"))
Expect(m.Suffix()).To(Equal("mp3"))
Expect(m.Size()).To(Equal(60845))
})
It("correctly parses ogg file with no tags", func() {
m, err := ExtractMetadata("../tests/fixtures/test.ogg")
Expect(err).To(BeNil())
Expect(m.Title()).To(BeEmpty())
Expect(m.HasPicture()).To(BeFalse())
Expect(m.Duration()).To(Equal(3))
Expect(m.BitRate()).To(Equal(9))
Expect(m.Suffix()).To(Equal("ogg"))
Expect(m.FilePath()).To(Equal("../tests/fixtures/test.ogg"))
Expect(m.Size()).To(Equal(4408))
})
It("returns error for invalid media file", func() {
_, err := ExtractMetadata("../tests/fixtures/itunes-library.xml")
Expect(err).ToNot(BeNil())
})
It("returns error for file not found", func() {
_, err := ExtractMetadata("../tests/fixtures/NOT-FOUND.mp3")
Expect(err).ToNot(BeNil())
})
})

121
scanner/scanner.go Normal file
View File

@ -0,0 +1,121 @@
package scanner
import (
"context"
"errors"
"fmt"
"strconv"
"time"
"github.com/cloudsonic/sonic-server/log"
"github.com/cloudsonic/sonic-server/model"
)
type Scanner struct {
folders map[string]FolderScanner
repos Repositories
}
type Repositories struct {
folder model.MediaFolderRepository
mediaFile model.MediaFileRepository
album model.AlbumRepository
artist model.ArtistRepository
index model.ArtistIndexRepository
playlist model.PlaylistRepository
property model.PropertyRepository
}
func New(mfRepo model.MediaFileRepository, albumRepo model.AlbumRepository, artistRepo model.ArtistRepository, idxRepo model.ArtistIndexRepository, plsRepo model.PlaylistRepository, folderRepo model.MediaFolderRepository, property model.PropertyRepository) *Scanner {
repos := Repositories{
folder: folderRepo,
mediaFile: mfRepo,
album: albumRepo,
artist: artistRepo,
index: idxRepo,
playlist: plsRepo,
property: property,
}
s := &Scanner{repos: repos, folders: map[string]FolderScanner{}}
s.loadFolders()
return s
}
func (s *Scanner) Rescan(mediaFolder string, fullRescan bool) error {
folderScanner := s.folders[mediaFolder]
start := time.Now()
lastModifiedSince := time.Time{}
if !fullRescan {
lastModifiedSince = s.getLastModifiedSince(mediaFolder)
log.Debug("Scanning folder", "folder", mediaFolder, "lastModifiedSince", lastModifiedSince)
} else {
log.Debug("Scanning folder (full scan)", "folder", mediaFolder)
}
err := folderScanner.Scan(nil, lastModifiedSince)
if err != nil {
log.Error("Error importing MediaFolder", "folder", mediaFolder, err)
}
s.updateLastModifiedSince(mediaFolder, start)
log.Debug("Finished scanning folder", "folder", mediaFolder, "elapsed", time.Since(start))
return err
}
func (s *Scanner) RescanAll(fullRescan bool) error {
var hasError bool
for folder := range s.folders {
err := s.Rescan(folder, fullRescan)
hasError = hasError || err != nil
}
if hasError {
log.Error("Errors while scanning media. Please check the logs")
return errors.New("errors while scanning media")
}
return nil
}
func (s *Scanner) Status() []StatusInfo { return nil }
func (i *Scanner) getLastModifiedSince(folder string) time.Time {
ms, err := i.repos.property.Get(model.PropLastScan + "-" + folder)
if err != nil {
return time.Time{}
}
if ms == "" {
return time.Time{}
}
s, _ := strconv.ParseInt(ms, 10, 64)
return time.Unix(0, s*int64(time.Millisecond))
}
func (s *Scanner) updateLastModifiedSince(folder string, t time.Time) {
millis := t.UnixNano() / int64(time.Millisecond)
s.repos.property.Put(model.PropLastScan+"-"+folder, fmt.Sprint(millis))
}
func (s *Scanner) loadFolders() {
fs, _ := s.repos.folder.GetAll()
for _, f := range fs {
log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path)
s.folders[f.Path] = NewTagScanner(f.Path, s.repos)
}
}
type Status int
const (
StatusComplete Status = iota
StatusInProgress
StatusError
)
type StatusInfo struct {
MediaFolder string
Status Status
}
type FolderScanner interface {
Scan(ctx context.Context, lastModifiedSince time.Time) error
}

36
scanner/scanner_suite_test.go Normal file
View File

@ -0,0 +1,36 @@
package scanner
import (
"testing"
"time"
"github.com/cloudsonic/sonic-server/conf"
"github.com/cloudsonic/sonic-server/log"
"github.com/cloudsonic/sonic-server/persistence"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestScanner(t *testing.T) {
log.SetLevel(log.LevelCritical)
RegisterFailHandler(Fail)
RunSpecs(t, "Scanner Suite")
}
var _ = XDescribe("TODO: REMOVE", func() {
conf.Sonic.DbPath = "./testDB"
log.SetLevel(log.LevelDebug)
repos := Repositories{
folder: persistence.NewMediaFolderRepository(),
mediaFile: persistence.NewMediaFileRepository(),
album: persistence.NewAlbumRepository(),
artist: persistence.NewArtistRepository(),
index: persistence.NewArtistIndexRepository(),
playlist: nil,
}
It("WORKS!", func() {
t := NewTagScanner("/Users/deluan/Music/iTunes/iTunes Media/Music", repos)
//t := NewTagScanner("/Users/deluan/Development/cloudsonic/sonic-server/tests/fixtures", repos)
Expect(t.Scan(nil, time.Time{})).To(BeNil())
})
})

273
scanner/tag_scanner.go Normal file
View File

@ -0,0 +1,273 @@
package scanner
import (
"context"
"crypto/md5"
"fmt"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/cloudsonic/sonic-server/log"
"github.com/cloudsonic/sonic-server/model"
)
type TagScanner struct {
rootFolder string
repos Repositories
}
func NewTagScanner(rootFolder string, repos Repositories) *TagScanner {
return &TagScanner{
rootFolder: rootFolder,
repos: repos,
}
}
// Scan algorithm overview:
// For each changed: Get all files from DB that starts with it, scan each file:
// if changed or new, delete from DB and add new from the file
// if not found, delete from DB
// scan and add the new ones
// For each deleted: delete all files from DB that starts with it
// Create new albums/artists, update counters (how?)
// collect all albumids and artistids from previous steps
// run something like this (for albums):
// select album_id, album, f.artist, f.compilation, max(f.year), count(*), sum(f.play_count), max(f.updated_at), a.id from media_file f left outer join album a on f.album_id = a.id group by album_id;
// when a.id is not null update, else insert (collect all inserts and run just one InsertMulti)
// Delete all empty albums, delete all empty Artists
// Recreate ArtistIndex
func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error {
detector := NewChangeDetector(s.rootFolder, lastModifiedSince)
changed, deleted, err := detector.Scan()
if err != nil {
return err
}
if len(changed)+len(deleted) == 0 {
return nil
}
log.Info("Folder changes found", "changed", len(changed), "deleted", len(deleted))
updatedArtists := map[string]bool{}
updatedAlbums := map[string]bool{}
for _, c := range changed {
err := s.processChangedDir(c, updatedArtists, updatedAlbums)
if err != nil {
return err
}
}
for _, c := range deleted {
err := s.processDeletedDir(c, updatedArtists, updatedAlbums)
if err != nil {
return err
}
}
err = s.refreshAlbums(updatedAlbums)
if err != nil {
return err
}
err = s.refreshArtists(updatedArtists)
if err != nil {
return err
}
err = s.repos.index.Refresh()
if err != nil {
return err
}
return nil
}
func (s *TagScanner) refreshAlbums(updatedAlbums map[string]bool) error {
var ids []string
for id := range updatedAlbums {
ids = append(ids, id)
}
return s.repos.album.Refresh(ids...)
}
func (s *TagScanner) refreshArtists(updatedArtists map[string]bool) error {
var ids []string
for id := range updatedArtists {
ids = append(ids, id)
}
return s.repos.artist.Refresh(ids...)
}
func (s *TagScanner) processChangedDir(dir string, updatedArtists map[string]bool, updatedAlbums map[string]bool) error {
dir = path.Join(s.rootFolder, dir)
start := time.Now()
// Load folder's current tracks from DB into a map
currentTracks := map[string]model.MediaFile{}
ct, err := s.repos.mediaFile.FindByPath(dir)
if err != nil {
return err
}
for _, t := range ct {
currentTracks[t.ID] = t
updatedArtists[t.ArtistID] = true
updatedAlbums[t.AlbumID] = true
}
// Load tracks from the folder
newTracks, err := s.loadTracks(dir)
if err != nil {
return err
}
// If track from folder is newer than the one in DB, update/insert in DB and delete from the current tracks
log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(newTracks))
numUpdatedTracks := 0
numPurgedTracks := 0
for _, n := range newTracks {
c, ok := currentTracks[n.ID]
if !ok || (ok && n.UpdatedAt.After(c.UpdatedAt)) {
err := s.repos.mediaFile.Put(&n)
updatedArtists[n.ArtistID] = true
updatedAlbums[n.AlbumID] = true
numUpdatedTracks++
if err != nil {
return err
}
}
delete(currentTracks, n.ID)
}
// Remaining tracks from DB that are not in the folder are deleted
for id := range currentTracks {
numPurgedTracks++
if err := s.repos.mediaFile.Delete(id); err != nil {
return err
}
}
log.Debug("Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, "purged", numPurgedTracks, "elapsed", time.Since(start))
return nil
}
func (s *TagScanner) processDeletedDir(dir string, updatedArtists map[string]bool, updatedAlbums map[string]bool) error {
dir = path.Join(s.rootFolder, dir)
ct, err := s.repos.mediaFile.FindByPath(dir)
if err != nil {
return err
}
for _, t := range ct {
updatedArtists[t.ArtistID] = true
updatedAlbums[t.AlbumID] = true
}
return s.repos.mediaFile.DeleteByPath(dir)
}
func (s *TagScanner) loadTracks(dirPath string) (model.MediaFiles, error) {
dir, err := os.Open(dirPath)
if err != nil {
return nil, err
}
files, err := dir.Readdir(-1)
if err != nil {
return nil, err
}
var mds model.MediaFiles
for _, f := range files {
if f.IsDir() {
continue
}
filePath := path.Join(dirPath, f.Name())
md, err := ExtractMetadata(filePath)
if err != nil {
continue
}
mf := s.toMediaFile(md)
mds = append(mds, mf)
}
return mds, nil
}
func (s *TagScanner) toMediaFile(md *Metadata) model.MediaFile {
mf := model.MediaFile{}
mf.ID = s.trackID(md)
mf.Title = s.mapTrackTitle(md)
mf.Album = md.Album()
mf.AlbumID = s.albumID(md)
mf.Album = s.mapAlbumName(md)
if md.Artist() == "" {
mf.Artist = "[Unknown Artist]"
} else {
mf.Artist = md.Artist()
}
mf.ArtistID = s.artistID(md)
mf.AlbumArtist = md.AlbumArtist()
mf.Genre = md.Genre()
mf.Compilation = md.Compilation()
mf.Year = md.Year()
mf.TrackNumber, _ = md.TrackNumber()
mf.DiscNumber, _ = md.DiscNumber()
mf.Duration = md.Duration()
mf.BitRate = md.BitRate()
mf.Path = md.FilePath()
mf.Suffix = md.Suffix()
mf.Size = strconv.Itoa(md.Size())
mf.HasCoverArt = md.HasPicture()
// TODO Get Creation time. https://github.com/djherbis/times ?
mf.CreatedAt = md.ModificationTime()
mf.UpdatedAt = md.ModificationTime()
return mf
}
func (s *TagScanner) mapTrackTitle(md *Metadata) string {
if md.Title() == "" {
s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
e := filepath.Ext(s)
return strings.TrimSuffix(s, e)
}
return md.Title()
}
func (s *TagScanner) mapArtistName(md *Metadata) string {
switch {
case md.Compilation():
return "Various Artists"
case md.AlbumArtist() != "":
return md.AlbumArtist()
case md.Artist() != "":
return md.Artist()
default:
return "[Unknown Artist]"
}
}
func (s *TagScanner) mapAlbumName(md *Metadata) string {
name := md.Album()
if name == "" {
return "[Unknown Album]"
}
return name
}
func (s *TagScanner) trackID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
}
func (s *TagScanner) albumID(md *Metadata) string {
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapArtistName(md), s.mapAlbumName(md)))
return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
}
func (s *TagScanner) artistID(md *Metadata) string {
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
}

50
server/app.go
View File

@ -9,6 +9,7 @@ import (
"github.com/cloudsonic/sonic-server/conf"
"github.com/cloudsonic/sonic-server/log"
"github.com/cloudsonic/sonic-server/scanner"
"github.com/cloudsonic/sonic-server/scanner_legacy"
"github.com/go-chi/chi"
"github.com/go-chi/chi/middleware"
@ -19,17 +20,24 @@ const Version = "0.2"
type Server struct {
Importer *scanner_legacy.Importer
Scanner *scanner.Scanner
router *chi.Mux
}
func New(importer *scanner_legacy.Importer) *Server {
a := &Server{Importer: importer}
func New(importer *scanner_legacy.Importer, scanner *scanner.Scanner) *Server {
a := &Server{Importer: importer, Scanner: scanner}
if !conf.Sonic.DevDisableBanner {
showBanner(Version)
}
initMimeTypes()
a.initRoutes()
a.initImporter()
if conf.Sonic.DevUseFileScanner {
log.Info("Using Folder Scanner", "folder", conf.Sonic.MusicFolder)
a.initScanner()
} else {
log.Info("Using iTunes Importer", "xml", conf.Sonic.MusicFolder)
a.initImporter()
}
return a
}
@ -67,22 +75,34 @@ func (a *Server) initRoutes() {
a.router = r
}
func (a *Server) initImporter() {
go a.startPeriodicScans()
func (a *Server) initScanner() {
go func() {
for {
select {
case <-time.After(5 * time.Second):
err := a.Scanner.RescanAll(false)
if err != nil {
log.Error("Error scanning media folder", "folder", conf.Sonic.MusicFolder, err)
}
}
}
}()
}
func (a *Server) startPeriodicScans() {
first := true
for {
select {
case <-time.After(5 * time.Second):
if first {
log.Info("Started iTunes scanner", "xml", conf.Sonic.MusicFolder)
first = false
func (a *Server) initImporter() {
go func() {
first := true
for {
select {
case <-time.After(5 * time.Second):
if first {
log.Info("Started iTunes scanner", "xml", conf.Sonic.MusicFolder)
first = false
}
a.Importer.CheckForUpdates(false)
}
a.Importer.CheckForUpdates(false)
}
}
}()
}
func FileServer(r chi.Router, path string, root http.FileSystem) {

BIN
tests/fixtures/test.mp3 vendored Normal file
View File

Binary file not shown.

BIN
tests/fixtures/test.ogg vendored Normal file
View File

Binary file not shown.

7
wire_gen.go
View File

@ -10,6 +10,7 @@ import (
"github.com/cloudsonic/sonic-server/engine"
"github.com/cloudsonic/sonic-server/itunesbridge"
"github.com/cloudsonic/sonic-server/persistence"
"github.com/cloudsonic/sonic-server/scanner"
"github.com/cloudsonic/sonic-server/scanner_legacy"
"github.com/cloudsonic/sonic-server/server"
"github.com/google/wire"
@ -27,7 +28,9 @@ func CreateApp(musicFolder string) *server.Server {
playlistRepository := persistence.NewPlaylistRepository()
propertyRepository := persistence.NewPropertyRepository()
importer := scanner_legacy.NewImporter(musicFolder, itunesScanner, mediaFileRepository, albumRepository, artistRepository, artistIndexRepository, playlistRepository, propertyRepository)
serverServer := server.New(importer)
mediaFolderRepository := persistence.NewMediaFolderRepository()
scannerScanner := scanner.New(mediaFileRepository, albumRepository, artistRepository, artistIndexRepository, playlistRepository, mediaFolderRepository, propertyRepository)
serverServer := server.New(importer, scannerScanner)
return serverServer
}
@ -55,4 +58,4 @@ func CreateSubsonicAPIRouter() *api.Router {
// wire_injectors.go:
var allProviders = wire.NewSet(itunesbridge.NewItunesControl, engine.Set, scanner_legacy.Set, api.NewRouter, persistence.Set)
var allProviders = wire.NewSet(itunesbridge.NewItunesControl, engine.Set, scanner_legacy.Set, scanner.New, api.NewRouter, persistence.Set)

2
wire_injectors.go
View File

@ -7,6 +7,7 @@ import (
"github.com/cloudsonic/sonic-server/engine"
"github.com/cloudsonic/sonic-server/itunesbridge"
"github.com/cloudsonic/sonic-server/persistence"
"github.com/cloudsonic/sonic-server/scanner"
"github.com/cloudsonic/sonic-server/scanner_legacy"
"github.com/cloudsonic/sonic-server/server"
"github.com/google/wire"
@ -16,6 +17,7 @@ var allProviders = wire.NewSet(
itunesbridge.NewItunesControl,
engine.Set,
scanner_legacy.Set,
scanner.New,
api.NewRouter,
persistence.Set,
)