mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Extract metadata from files in a directory in batch (one call to ffmpeg for all files)

This commit is contained in:
Deluan 2020-01-21 00:49:41 -05:00
parent d9ce6f3b51
commit c42e1bd2f5
7 changed files with 139 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
conf/configuration.go
View File

@ -17,16 +17,16 @@ type sonic struct {
DisableDownsampling bool `default:"false"`
DownsampleCommand string `default:"ffmpeg -i %s -map 0:0 -b:a %bk -v 0 -f mp3 -"`
//ProbeCommand string `default:"ffprobe -v quiet -print_format json -show_format %s"`
ProbeCommand string `default:"ffmpeg -i %s -f ffmetadata"`
PlsIgnoreFolders bool `default:"true"`
PlsIgnoredPatterns string `default:"^iCloud;\\~"`
ProbeCommand string `default:"ffmpeg %s -f ffmetadata"`
PlsIgnoreFolders bool `default:"true"`
PlsIgnoredPatterns string `default:"^iCloud;\\~"`
// DevFlags
LogLevel string `default:"info"`
DevDisableAuthentication bool `default:"false"`
DevDisableFileCheck bool `default:"false"`
DevDisableBanner bool `default:"false"`
DevInitialPassword string `default:""`
}
var Sonic *sonic

159
scanner/metadata_ffmpeg.go
View File

@ -2,7 +2,6 @@ package scanner
import (
"bufio"
"bytes"
"errors"
"mime"
"os"
@ -24,31 +23,6 @@ type Metadata struct {
tags map[string]string
}
func ExtractMetadata(filePath string) (*Metadata, error) {
m := &Metadata{filePath: filePath, tags: map[string]string{}}
extension := path.Ext(filePath)
if !isAudioFile(extension) {
return nil, errors.New("not an audio file")
}
m.suffix = strings.ToLower(strings.TrimPrefix(extension, "."))
fi, err := os.Stat(filePath)
if err != nil {
return nil, err
}
m.fileInfo = fi
err = m.probe(filePath)
if len(m.tags) == 0 {
return nil, errors.New("not a media file")
}
return m, err
}
func isAudioFile(extension string) bool {
typ := mime.TypeByExtension(extension)
return strings.HasPrefix(typ, "audio/")
}
func (m *Metadata) Title() string { return m.tags["title"] }
func (m *Metadata) Album() string { return m.tags["album"] }
func (m *Metadata) Artist() string { return m.tags["artist"] }
@ -67,40 +41,116 @@ func (m *Metadata) FilePath() string { return m.filePath }
func (m *Metadata) Suffix() string { return m.suffix }
func (m *Metadata) Size() int { return int(m.fileInfo.Size()) }
func (m *Metadata) probe(filePath string) error {
cmdLine, args := createProbeCommand(filePath)
func ExtractAllMetadata(dirPath string) (map[string]*Metadata, error) {
dir, err := os.Open(dirPath)
if err != nil {
return nil, err
}
files, err := dir.Readdir(-1)
if err != nil {
return nil, err
}
var audioFiles []string
for _, f := range files {
if f.IsDir() {
continue
}
filePath := path.Join(dirPath, f.Name())
extension := path.Ext(filePath)
if !isAudioFile(extension) {
continue
}
audioFiles = append(audioFiles, filePath)
}
if len(audioFiles) == 0 {
return map[string]*Metadata{}, nil
}
return probe(audioFiles)
}
func probe(inputs []string) (map[string]*Metadata, error) {
cmdLine, args := createProbeCommand(inputs)
log.Trace("Executing command", "cmdLine", cmdLine, "args", args)
cmd := exec.Command(cmdLine, args...)
output, _ := cmd.CombinedOutput()
if len(output) == 0 || bytes.Contains(output, []byte("No such file or directory")) {
return errors.New("error extracting metadata from " + filePath)
mds := map[string]*Metadata{}
if len(output) == 0 {
return mds, errors.New("error extracting metadata files")
}
return m.parseOutput(output)
infos := parseOutput(string(output))
for file, info := range infos {
md, err := extractMetadata(file, info)
if err == nil {
mds[file] = md
}
}
return mds, nil
}
var inputRegex = regexp.MustCompile(`(?m)^Input #\d+,.*,\sfrom\s'(.*)'`)
func parseOutput(output string) map[string]string {
split := map[string]string{}
all := inputRegex.FindAllStringSubmatchIndex(string(output), -1)
for i, loc := range all {
// Filename is the first captured group
file := output[loc[2]:loc[3]]
// File info is everything from the match, up until the beginning of the next match
info := ""
initial := loc[1]
if i < len(all)-1 {
end := all[i+1][0] - 1
info = output[initial:end]
} else {
// if this is the last match
info = output[initial:]
}
split[file] = info
}
return split
}
func extractMetadata(filePath, info string) (*Metadata, error) {
m := &Metadata{filePath: filePath, tags: map[string]string{}}
m.suffix = strings.ToLower(strings.TrimPrefix(path.Ext(filePath), "."))
m.parseInfo(info)
m.fileInfo, _ = os.Stat(filePath)
if len(m.tags) == 0 {
return nil, errors.New("not a media file")
}
return m, nil
}
func isAudioFile(extension string) bool {
typ := mime.TypeByExtension(extension)
return strings.HasPrefix(typ, "audio/")
}
var (
tagsRx = map[*regexp.Regexp]string{
regexp.MustCompile(`^\s+compilation\s+:(.*)`): "compilation",
regexp.MustCompile(`^\s+genre\s+:\s(.*)`): "genre",
regexp.MustCompile(`^\s+title\s+:\s(.*)`): "title",
regexp.MustCompile(`^\s{4}comment\s+:\s(.*)`): "comment",
regexp.MustCompile(`^\s+artist\s+:\s(.*)`): "artist",
regexp.MustCompile(`^\s+album_artist\s+:\s(.*)`): "album_artist",
regexp.MustCompile(`^\s+TCM\s+:\s(.*)`): "composer",
regexp.MustCompile(`^\s+album\s+:\s(.*)`): "album",
regexp.MustCompile(`^\s+track\s+:\s(.*)`): "trackNum",
regexp.MustCompile(`^\s+disc\s+:\s(.*)`): "discNum",
regexp.MustCompile(`^\s+TPA\s+:\s(.*)`): "discNum",
regexp.MustCompile(`^\s+date\s+:\s(.*)`): "year",
regexp.MustCompile(`^\s{4}Stream #0:1: (.+)\:\s`): "hasPicture",
regexp.MustCompile(`^\s+compilation\s+:(.*)`): "compilation",
regexp.MustCompile(`^\s+genre\s+:\s(.*)`): "genre",
regexp.MustCompile(`^\s+title\s+:\s(.*)`): "title",
regexp.MustCompile(`^\s{4}comment\s+:\s(.*)`): "comment",
regexp.MustCompile(`^\s+artist\s+:\s(.*)`): "artist",
regexp.MustCompile(`^\s+album_artist\s+:\s(.*)`): "album_artist",
regexp.MustCompile(`^\s+TCM\s+:\s(.*)`): "composer",
regexp.MustCompile(`^\s+album\s+:\s(.*)`): "album",
regexp.MustCompile(`^\s+track\s+:\s(.*)`): "trackNum",
regexp.MustCompile(`^\s+disc\s+:\s(.*)`): "discNum",
regexp.MustCompile(`^\s+TPA\s+:\s(.*)`): "discNum",
regexp.MustCompile(`^\s+date\s+:\s(.*)`): "year",
regexp.MustCompile(`^\s{4}Stream #\d+:1: (.+):\s`): "hasPicture",
}
durationRx = regexp.MustCompile(`^\s\sDuration: ([\d.:]+).*bitrate: (\d+)`)
)
func (m *Metadata) parseOutput(output []byte) error {
reader := strings.NewReader(string(output))
func (m *Metadata) parseInfo(info string) {
reader := strings.NewReader(info)
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
@ -120,7 +170,6 @@ func (m *Metadata) parseOutput(output []byte) error {
}
}
}
return nil
}
func (m *Metadata) parseInt(tagName string) int {
@ -165,14 +214,20 @@ func (m *Metadata) parseDuration(tagName string) int {
return 0
}
func createProbeCommand(filePath string) (string, []string) {
func createProbeCommand(inputs []string) (string, []string) {
cmd := conf.Sonic.ProbeCommand
split := strings.Split(cmd, " ")
for i, s := range split {
s = strings.Replace(s, "%s", filePath, -1)
split[i] = s
args := make([]string, 0)
for _, s := range split {
if s == "%s" {
for _, inp := range inputs {
args = append(args, "-i", inp)
}
continue
}
args = append(args, s)
}
return split[0], split[1:]
return args[0], args[1:]
}

142
scanner/metadata_ffprobe.go
View File

@ -1,142 +0,0 @@
//+build ignored
package scanner
import (
"encoding/json"
"errors"
"os"
"os/exec"
"path"
"strconv"
"strings"
"time"
"github.com/cloudsonic/sonic-server/conf"
"github.com/cloudsonic/sonic-server/log"
"github.com/dhowden/tag"
)
type Metadata struct {
filePath string
suffix string
fileInfo os.FileInfo
t tag.Metadata
duration int
bitRate int
compilation bool
}
func ExtractMetadata(filePath string) (*Metadata, error) {
m := &Metadata{filePath: filePath}
m.suffix = strings.ToLower(strings.TrimPrefix(path.Ext(filePath), "."))
fi, err := os.Stat(filePath)
if err != nil {
return nil, err
}
m.fileInfo = fi
f, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer f.Close()
t, err := tag.ReadFrom(f)
if err != nil {
return nil, err
}
m.t = t
err = m.probe(filePath)
return m, err
}
func (m *Metadata) Title() string { return m.t.Title() }
func (m *Metadata) Album() string { return m.t.Album() }
func (m *Metadata) Artist() string { return m.t.Artist() }
func (m *Metadata) AlbumArtist() string { return m.t.AlbumArtist() }
func (m *Metadata) Composer() string { return m.t.Composer() }
func (m *Metadata) Genre() string { return m.t.Genre() }
func (m *Metadata) Year() int { return m.t.Year() }
func (m *Metadata) TrackNumber() (int, int) { return m.t.Track() }
func (m *Metadata) DiscNumber() (int, int) { return m.t.Disc() }
func (m *Metadata) HasPicture() bool { return m.t.Picture() != nil }
func (m *Metadata) Compilation() bool { return m.compilation }
func (m *Metadata) Duration() int { return m.duration }
func (m *Metadata) BitRate() int { return m.bitRate }
func (m *Metadata) ModificationTime() time.Time { return m.fileInfo.ModTime() }
func (m *Metadata) FilePath() string { return m.filePath }
func (m *Metadata) Suffix() string { return m.suffix }
func (m *Metadata) Size() int { return int(m.fileInfo.Size()) }
// probe analyzes the file and returns duration in seconds and bitRate in kb/s.
// It uses the ffprobe external tool, configured in conf.Sonic.ProbeCommand
func (m *Metadata) probe(filePath string) error {
cmdLine, args := createProbeCommand(filePath)
log.Trace("Executing command", "cmdLine", cmdLine, "args", args)
cmd := exec.Command(cmdLine, args...)
output, err := cmd.CombinedOutput()
if err != nil {
return err
}
return m.parseOutput(output)
}
func (m *Metadata) parseInt(objItf interface{}, field string) (int, error) {
obj := objItf.(map[string]interface{})
s, ok := obj[field].(string)
if !ok {
return -1, errors.New("invalid ffprobe output field obj." + field)
}
fDuration, err := strconv.ParseFloat(s, 64)
if err != nil {
return -1, err
}
return int(fDuration), nil
}
func (m *Metadata) parseOutput(output []byte) error {
var data map[string]map[string]interface{}
err := json.Unmarshal(output, &data)
if err != nil {
return err
}
format, ok := data["format"]
if !ok {
err = errors.New("invalid ffprobe output. no format found")
return err
}
if tags, ok := format["tags"]; ok {
c, _ := m.parseInt(tags, "compilation")
m.compilation = c == 1
}
m.duration, err = m.parseInt(format, "duration")
if err != nil {
return err
}
m.bitRate, err = m.parseInt(format, "bit_rate")
m.bitRate = m.bitRate / 1000
if err != nil {
return err
}
return nil
}
func createProbeCommand(filePath string) (string, []string) {
cmd := conf.Sonic.ProbeCommand
split := strings.Split(cmd, " ")
for i, s := range split {
s = strings.Replace(s, "%s", filePath, -1)
split[i] = s
}
return split[0], split[1:]
}

24
scanner/metadata_test.go
View File

@ -6,9 +6,12 @@ import (
)
var _ = Describe("Metadata", func() {
FIt("correctly parses mp3 file", func() {
m, err := ExtractMetadata("../tests/fixtures/test.mp3")
Expect(err).To(BeNil())
It("correctly parses metadata from all files in folder", func() {
mds, err := ExtractAllMetadata("../tests/fixtures")
Expect(err).NotTo(HaveOccurred())
Expect(mds).To(HaveLen(3))
m := mds["../tests/fixtures/test.mp3"]
Expect(m.Title()).To(Equal("Song"))
Expect(m.Album()).To(Equal("Album"))
Expect(m.Artist()).To(Equal("Artist"))
@ -29,10 +32,8 @@ var _ = Describe("Metadata", func() {
Expect(m.FilePath()).To(Equal("../tests/fixtures/test.mp3"))
Expect(m.Suffix()).To(Equal("mp3"))
Expect(m.Size()).To(Equal(60845))
})
It("correctly parses ogg file with no tags", func() {
m, err := ExtractMetadata("../tests/fixtures/test.ogg")
m = mds["../tests/fixtures/test.ogg"]
Expect(err).To(BeNil())
Expect(m.Title()).To(BeEmpty())
Expect(m.HasPicture()).To(BeFalse())
@ -43,13 +44,12 @@ var _ = Describe("Metadata", func() {
Expect(m.Size()).To(Equal(4408))
})
FIt("returns error for invalid media file", func() {
_, err := ExtractMetadata("../tests/fixtures/itunes-library.xml")
Expect(err).ToNot(BeNil())
It("returns error if path does not exist", func() {
_, err := ExtractAllMetadata("./INVALID/PATH")
Expect(err).To(HaveOccurred())
})
It("returns error for file not found", func() {
_, err := ExtractMetadata("../tests/fixtures/NOT-FOUND.mp3")
Expect(err).ToNot(BeNil())
It("returns empty map if there are no audio files in path", func() {
Expect(ExtractAllMetadata(".")).To(BeEmpty())
})
})

7
scanner/scanner_suite_test.go
View File

@ -19,10 +19,11 @@ func xTestScanner(t *testing.T) {
}
var _ = XDescribe("TODO: REMOVE", func() {
conf.Sonic.DbPath = "./testDB"
log.SetLevel(log.LevelDebug)
ds := persistence.New()
It("WORKS!", func() {
conf.Sonic.DbPath = "./testDB"
log.SetLevel(log.LevelDebug)
ds := persistence.New()
t := NewTagScanner("/Users/deluan/Music/iTunes/iTunes Media/Music", ds)
//t := NewTagScanner("/Users/deluan/Development/cloudsonic/sonic-server/tests/fixtures", ds)
Expect(t.Scan(nil, time.Time{})).To(BeNil())

22
scanner/tag_scanner.go
View File

@ -208,28 +208,16 @@ func (s *TagScanner) processDeletedDir(dir string, updatedArtists map[string]boo
}
func (s *TagScanner) loadTracks(dirPath string) (model.MediaFiles, error) {
dir, err := os.Open(dirPath)
mds, err := ExtractAllMetadata(dirPath)
if err != nil {
return nil, err
}
files, err := dir.Readdir(-1)
if err != nil {
return nil, err
}
var mds model.MediaFiles
for _, f := range files {
if f.IsDir() {
continue
}
filePath := path.Join(dirPath, f.Name())
md, err := ExtractMetadata(filePath)
if err != nil {
continue
}
var mfs model.MediaFiles
for _, md := range mds {
mf := s.toMediaFile(md)
mds = append(mds, mf)
mfs = append(mfs, mf)
}
return mds, nil
return mfs, nil
}
func (s *TagScanner) toMediaFile(md *Metadata) model.MediaFile {

9
server/initial_setup.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"time"
"github.com/cloudsonic/sonic-server/conf"
"github.com/cloudsonic/sonic-server/consts"
"github.com/cloudsonic/sonic-server/log"
"github.com/cloudsonic/sonic-server/model"
@ -50,14 +51,18 @@ func createDefaultUser(ds model.DataStore) error {
}
if c == 0 {
id, _ := uuid.NewRandom()
initialPassword, _ := uuid.NewRandom()
random, _ := uuid.NewRandom()
initialPassword := random.String()
if conf.Sonic.DevInitialPassword != "" {
initialPassword = conf.Sonic.DevInitialPassword
}
log.Warn("Creating initial user. Please change the password!", "user", consts.InitialUserName, "password", initialPassword)
initialUser := model.User{
ID: id.String(),
UserName: consts.InitialUserName,
Name: consts.InitialName,
Email: "",
Password: initialPassword.String(),
Password: initialPassword,
IsAdmin: true,
}
err := ds.User().Put(&initialUser)