package scanner import ( "crypto/md5" "fmt" "html" "mime" "net/url" "os" "path/filepath" "regexp" "strconv" "strings" "time" "github.com/astaxie/beego" "github.com/cloudsonic/sonic-server/conf" "github.com/cloudsonic/sonic-server/domain" "github.com/dhowden/itl" "github.com/dhowden/tag" ) type ItunesScanner struct { mediaFiles map[string]*domain.MediaFile albums map[string]*domain.Album artists map[string]*domain.Artist playlists map[string]*domain.Playlist pplaylists map[string]plsRelation pmediaFiles map[int]*domain.MediaFile lastModifiedSince time.Time checksumRepo CheckSumRepository newSums map[string]string } func NewItunesScanner(checksumRepo CheckSumRepository) *ItunesScanner { return &ItunesScanner{checksumRepo: checksumRepo} } type CheckSumRepository interface { Put(id, sum string) error Get(id string) (string, error) SetData(newSums map[string]string) error } type plsRelation struct { pID string parentPID string name string } func (s *ItunesScanner) ScanLibrary(lastModifiedSince time.Time, path string) (int, error) { beego.Debug("Checking for updates since", lastModifiedSince.String(), "- Library:", path) xml, _ := os.Open(path) l, err := itl.ReadFromXML(xml) if err != nil { return 0, err } beego.Debug("Loaded", len(l.Tracks), "tracks") s.lastModifiedSince = lastModifiedSince s.mediaFiles = make(map[string]*domain.MediaFile) s.albums = make(map[string]*domain.Album) s.artists = make(map[string]*domain.Artist) s.playlists = make(map[string]*domain.Playlist) s.pplaylists = make(map[string]plsRelation) s.pmediaFiles = make(map[int]*domain.MediaFile) s.newSums = make(map[string]string) songsPerAlbum := make(map[string]int) albumsPerArtist := make(map[string]map[string]bool) i := 0 for _, t := range l.Tracks { if !s.skipTrack(&t) { s.calcCheckSum(&t) ar := s.collectArtists(&t) mf := s.collectMediaFiles(&t) s.collectAlbums(&t, mf, ar) songsPerAlbum[mf.AlbumId]++ if albumsPerArtist[mf.ArtistId] == nil { albumsPerArtist[mf.ArtistId] = make(map[string]bool) } albumsPerArtist[mf.ArtistId][mf.AlbumId] = true } i++ if i%1000 == 0 { beego.Debug("Processed", i, "tracks.", len(s.artists), "artists,", len(s.albums), "albums", len(s.mediaFiles), "songs") } } for albumId, count := range songsPerAlbum { s.albums[albumId].SongCount = count } for artistId, albums := range albumsPerArtist { s.artists[artistId].AlbumCount = len(albums) } if err := s.checksumRepo.SetData(s.newSums); err != nil { beego.Error("Error saving checksums:", err) } else { beego.Debug("Saved", len(s.newSums), "checksums") } ignFolders := conf.Sonic.PlsIgnoreFolders ignPatterns := strings.Split(conf.Sonic.PlsIgnoredPatterns, ";") for _, p := range l.Playlists { rel := plsRelation{pID: p.PlaylistPersistentID, parentPID: p.ParentPersistentID, name: unescape(p.Name)} s.pplaylists[p.PlaylistPersistentID] = rel fullPath := s.fullPath(p.PlaylistPersistentID) if s.skipPlaylist(&p, ignFolders, ignPatterns, fullPath) { continue } s.collectPlaylists(&p, fullPath) } beego.Debug("Processed", len(l.Playlists), "playlists.") return len(l.Tracks), nil } func (s *ItunesScanner) MediaFiles() map[string]*domain.MediaFile { return s.mediaFiles } func (s *ItunesScanner) Albums() map[string]*domain.Album { return s.albums } func (s *ItunesScanner) Artists() map[string]*domain.Artist { return s.artists } func (s *ItunesScanner) Playlists() map[string]*domain.Playlist { return s.playlists } func (s *ItunesScanner) skipTrack(t *itl.Track) bool { if !strings.HasPrefix(t.Location, "file://") || t.Podcast { return true } ext := filepath.Ext(t.Location) m := mime.TypeByExtension(ext) return !strings.HasPrefix(m, "audio/") } func (s *ItunesScanner) skipPlaylist(p *itl.Playlist, ignFolders bool, ignPatterns []string, fullPath string) bool { // Skip all "special" iTunes playlists, and also ignored patterns if p.Master || p.Music || p.Audiobooks || p.Movies || p.TVShows || p.Podcasts || p.ITunesU || (ignFolders && p.Folder) { return true } for _, p := range ignPatterns { if p == "" { continue } m, _ := regexp.MatchString(p, fullPath) if m { return true } } return false } func (s *ItunesScanner) collectPlaylists(p *itl.Playlist, fullPath string) { pl := &domain.Playlist{} pl.Id = p.PlaylistPersistentID pl.Name = unescape(p.Name) pl.FullPath = fullPath pl.Tracks = make([]string, 0, len(p.PlaylistItems)) for _, item := range p.PlaylistItems { if mf, found := s.pmediaFiles[item.TrackID]; found { pl.Tracks = append(pl.Tracks, mf.Id) pl.Duration += mf.Duration } } if len(pl.Tracks) > 0 { s.playlists[pl.Id] = pl } } func (s *ItunesScanner) fullPath(pID string) string { rel, found := s.pplaylists[pID] if !found { return "" } if rel.parentPID == "" { return rel.name } return fmt.Sprintf("%s > %s", s.fullPath(rel.parentPID), rel.name) } func (s *ItunesScanner) lastChangedDate(t *itl.Track) time.Time { if s.hasChanged(t) { return time.Now() } allDates := []time.Time{t.DateModified, t.PlayDateUTC} c := time.Time{} for _, d := range allDates { if c.Before(d) { c = d } } return c } func (s *ItunesScanner) hasChanged(t *itl.Track) bool { id := t.PersistentID oldSum, _ := s.checksumRepo.Get(id) newSum := s.newSums[id] return oldSum != newSum } // Calc sum of stats fields (whose changes are not reflected in DataModified) func (s *ItunesScanner) calcCheckSum(t *itl.Track) string { id := t.PersistentID data := fmt.Sprint(t.DateModified, t.PlayCount, t.PlayDate, t.ArtworkCount, t.Loved, t.AlbumLoved, t.Rating, t.AlbumRating, t.SkipCount, t.SkipDate) sum := fmt.Sprintf("%x", md5.Sum([]byte(data))) s.newSums[id] = sum return sum } func (s *ItunesScanner) collectMediaFiles(t *itl.Track) *domain.MediaFile { mf := &domain.MediaFile{} mf.Id = t.PersistentID mf.Album = unescape(t.Album) mf.AlbumId = albumId(t) mf.ArtistId = artistId(t) mf.Title = unescape(t.Name) mf.Artist = unescape(t.Artist) if mf.Album == "" { mf.Album = "[Unknown Album]" } if mf.Artist == "" { mf.Artist = "[Unknown Artist]" } mf.AlbumArtist = unescape(t.AlbumArtist) mf.Genre = unescape(t.Genre) mf.Compilation = t.Compilation mf.Starred = t.Loved mf.Rating = t.Rating / 20 mf.PlayCount = t.PlayCount mf.PlayDate = t.PlayDateUTC mf.Year = t.Year mf.TrackNumber = t.TrackNumber mf.DiscNumber = t.DiscNumber if t.Size > 0 { mf.Size = strconv.Itoa(t.Size) } if t.TotalTime > 0 { mf.Duration = t.TotalTime / 1000 } mf.BitRate = t.BitRate path := extractPath(t.Location) mf.Path = path mf.Suffix = strings.TrimPrefix(filepath.Ext(path), ".") mf.CreatedAt = t.DateAdded mf.UpdatedAt = s.lastChangedDate(t) if mf.UpdatedAt.After(s.lastModifiedSince) { mf.HasCoverArt = hasCoverArt(path) } s.mediaFiles[mf.Id] = mf s.pmediaFiles[t.TrackID] = mf return mf } func (s *ItunesScanner) collectAlbums(t *itl.Track, mf *domain.MediaFile, ar *domain.Artist) *domain.Album { id := albumId(t) _, found := s.albums[id] if !found { s.albums[id] = &domain.Album{} } al := s.albums[id] al.Id = id al.ArtistId = ar.Id al.Name = mf.Album al.Year = t.Year al.Compilation = t.Compilation al.Starred = t.AlbumLoved al.Rating = t.AlbumRating / 20 al.PlayCount += t.PlayCount al.Genre = mf.Genre al.Artist = mf.Artist al.AlbumArtist = ar.Name if al.Name == "" { al.Name = "[Unknown Album]" } if al.Artist == "" { al.Artist = "[Unknown Artist]" } al.Duration += mf.Duration if mf.HasCoverArt { al.CoverArtId = mf.Id al.CoverArtPath = mf.Path } if al.PlayDate.IsZero() || t.PlayDateUTC.After(al.PlayDate) { al.PlayDate = t.PlayDateUTC } if al.CreatedAt.IsZero() || t.DateAdded.Before(al.CreatedAt) { al.CreatedAt = t.DateAdded } trackUpdate := s.lastChangedDate(t) if al.UpdatedAt.IsZero() || trackUpdate.After(al.UpdatedAt) { al.UpdatedAt = trackUpdate } return al } func (s *ItunesScanner) collectArtists(t *itl.Track) *domain.Artist { id := artistId(t) _, found := s.artists[id] if !found { s.artists[id] = &domain.Artist{} } ar := s.artists[id] ar.Id = id ar.Name = unescape(realArtistName(t)) if ar.Name == "" { ar.Name = "[Unknown Artist]" } return ar } func albumId(t *itl.Track) string { s := strings.ToLower(fmt.Sprintf("%s\\%s", realArtistName(t), t.Album)) return fmt.Sprintf("%x", md5.Sum([]byte(s))) } func artistId(t *itl.Track) string { return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(realArtistName(t))))) } func hasCoverArt(path string) bool { defer func() { if r := recover(); r != nil { beego.Error("Reading tag for", path, "Panic:", r) } }() if _, err := os.Stat(path); err == nil { f, err := os.Open(path) if err != nil { beego.Warn("Error opening file", path, "-", err) return false } defer f.Close() m, err := tag.ReadFrom(f) if err != nil { beego.Warn("Error reading tag from file", path, "-", err) return false } return m.Picture() != nil } //beego.Warn("File not found:", path) return false } func unescape(str string) string { return html.UnescapeString(str) } func extractPath(loc string) string { path := strings.Replace(loc, "+", "%2B", -1) path, _ = url.QueryUnescape(path) path = html.UnescapeString(path) return strings.TrimPrefix(path, "file://") } func realArtistName(t *itl.Track) string { switch { case t.Compilation: return "Various Artists" case t.AlbumArtist != "": return t.AlbumArtist } return t.Artist } var _ Scanner = (*ItunesScanner)(nil)