mirrors
/
navidrome
mirror of https://github.com/navidrome/navidrome.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactor string utilities into its own package `str`

This commit is contained in:
Deluan 2024-06-05 22:09:27 -04:00
parent 46fc38bf61
commit abe5690018
16 changed files with 158 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/artwork/reader_artist.go
View File

@ -17,7 +17,7 @@ import (
"github.com/navidrome/navidrome/core"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/str"
)
type artistReader struct {
@ -56,7 +56,7 @@ func newArtistReader(ctx context.Context, artwork *artwork, artID model.ArtworkI
}
}
a.files = strings.Join(files, consts.Zwsp)
a.artistFolder = utils.LongestCommonPrefix(paths)
a.artistFolder = str.LongestCommonPrefix(paths)
if !strings.HasSuffix(a.artistFolder, string(filepath.Separator)) {
a.artistFolder, _ = filepath.Split(a.artistFolder)
}

24
core/external_metadata.go
View File

@ -19,6 +19,7 @@ import (
"github.com/navidrome/navidrome/utils"
. "github.com/navidrome/navidrome/utils/gg"
"github.com/navidrome/navidrome/utils/random"
"github.com/navidrome/navidrome/utils/str"
"golang.org/x/sync/errgroup"
)
@ -74,7 +75,7 @@ func (e *externalMetadata) getAlbum(ctx context.Context, id string) (*auxAlbum,
switch v := entity.(type) {
case *model.Album:
album.Album = *v
album.Name = clearName(v.Name)
album.Name = str.Clear(v.Name)
case *model.MediaFile:
return e.getAlbum(ctx, v.AlbumID)
default:
@ -164,7 +165,7 @@ func (e *externalMetadata) getArtist(ctx context.Context, id string) (*auxArtist
switch v := entity.(type) {
case *model.Artist:
artist.Artist = *v
artist.Name = clearName(v.Name)
artist.Name = str.Clear(v.Name)
case *model.MediaFile:
return e.getArtist(ctx, v.ArtistID)
case *model.Album:
@ -175,17 +176,6 @@ func (e *externalMetadata) getArtist(ctx context.Context, id string) (*auxArtist
return &artist, nil
}
// Replace some Unicode chars with their equivalent ASCII
func clearName(name string) string {
name = strings.ReplaceAll(name, "–", "-")
name = strings.ReplaceAll(name, "‐", "-")
name = strings.ReplaceAll(name, "“", `"`)
name = strings.ReplaceAll(name, "”", `"`)
name = strings.ReplaceAll(name, "‘", `'`)
name = strings.ReplaceAll(name, "’", `'`)
return name
}
func (e *externalMetadata) UpdateArtistInfo(ctx context.Context, id string, similarCount int, includeNotPresent bool) (*model.Artist, error) {
artist, err := e.refreshArtistInfo(ctx, id)
if err != nil {
@ -414,7 +404,7 @@ func (e *externalMetadata) findMatchingTrack(ctx context.Context, mbid string, a
squirrel.Eq{"artist_id": artistID},
squirrel.Eq{"album_artist_id": artistID},
},
squirrel.Like{"order_title": utils.SanitizeFieldForSorting(title)},
squirrel.Like{"order_title": str.SanitizeFieldForSorting(title)},
},
Sort: "starred desc, rating desc, year asc, compilation asc ",
Max: 1,
@ -434,11 +424,11 @@ func (e *externalMetadata) callGetURL(ctx context.Context, agent agents.ArtistUR
}
func (e *externalMetadata) callGetBiography(ctx context.Context, agent agents.ArtistBiographyRetriever, artist *auxArtist) {
bio, err := agent.GetArtistBiography(ctx, artist.ID, clearName(artist.Name), artist.MbzArtistID)
bio, err := agent.GetArtistBiography(ctx, artist.ID, str.Clear(artist.Name), artist.MbzArtistID)
if err != nil {
return
}
bio = utils.SanitizeText(bio)
bio = str.SanitizeText(bio)
bio = strings.ReplaceAll(bio, "\n", " ")
artist.Biography = strings.ReplaceAll(bio, "<a ", "<a target='_blank' ")
}
@ -514,7 +504,7 @@ func (e *externalMetadata) findArtistByName(ctx context.Context, artistName stri
}
artist := &auxArtist{
Artist: artists[0],
Name: clearName(artists[0].Name),
Name: str.Clear(artists[0].Name),
}
return artist, nil
}

4
db/migrations/20201213124814_add_all_artist_ids_to_album.go
View File

@ -5,7 +5,7 @@ import (
"database/sql"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/str"
"github.com/pressly/goose/v3"
)
@ -50,7 +50,7 @@ select a.id, a.name, a.artist_id, a.album_artist_id, group_concat(mf.artist_id,
if err != nil {
return err
}
all := utils.SanitizeStrings(artistId, albumArtistId, songArtistIds.String)
all := str.SanitizeStrings(artistId, albumArtistId, songArtistIds.String)
_, err = stmt.Exec(all, id)
if err != nil {
log.Error("Error setting album's artist_ids", "album", name, "albumId", id, err)

6
db/migrations/20211026191915_unescape_lyrics_and_comments.go
View File

@ -5,7 +5,7 @@ import (
"database/sql"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/str"
"github.com/pressly/goose/v3"
)
@ -33,8 +33,8 @@ func upUnescapeLyricsAndComments(_ context.Context, tx *sql.Tx) error {
return err
}
newComment := utils.SanitizeText(comment.String)
newLyrics := utils.SanitizeText(lyrics.String)
newComment := str.SanitizeText(comment.String)
newLyrics := str.SanitizeText(lyrics.String)
_, err = stmt.Exec(newComment, newLyrics, id)
if err != nil {
log.Error("Error unescaping media_file's lyrics and comments", "title", title, "id", id, err)

8
model/lyrics.go
View File

@ -8,7 +8,7 @@ import (
"strings"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/str"
)
type Line struct {
@ -36,7 +36,7 @@ var (
)
func ToLyrics(language, text string) (*Lyrics, error) {
text = utils.SanitizeText(text)
text = str.SanitizeText(text)
lines := strings.Split(text, "\n")
@ -67,7 +67,7 @@ func ToLyrics(language, text string) (*Lyrics, error) {
if idTag != nil {
switch idTag[1] {
case "ar":
artist = utils.SanitizeText(strings.TrimSpace(idTag[2]))
artist = str.SanitizeText(strings.TrimSpace(idTag[2]))
case "offset":
{
off, err := strconv.ParseInt(strings.TrimSpace(idTag[2]), 10, 64)
@ -78,7 +78,7 @@ func ToLyrics(language, text string) (*Lyrics, error) {
}
}
case "ti":
title = utils.SanitizeText(strings.TrimSpace(idTag[2]))
title = str.SanitizeText(strings.TrimSpace(idTag[2]))
}
continue

4
model/mediafile.go
View File

@ -12,8 +12,8 @@ import (
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
"github.com/navidrome/navidrome/utils/str"
)
type MediaFile struct {
@ -187,7 +187,7 @@ func (mfs MediaFiles) ToAlbum() Album {
a.Genre = slice.MostFrequent(a.Genres).Name
slices.SortFunc(a.Genres, func(a, b Genre) int { return cmp.Compare(a.ID, b.ID) })
a.Genres = slices.Compact(a.Genres)
a.FullText = " " + utils.SanitizeStrings(fullText...)
a.FullText = " " + str.SanitizeStrings(fullText...)
a = fixAlbumArtist(a, albumArtistIds)
songArtistIds = append(songArtistIds, a.AlbumArtistID, a.ArtistID)
slices.Sort(songArtistIds)

3
persistence/artist_repository.go
View File

@ -14,6 +14,7 @@ import (
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/str"
"github.com/pocketbase/dbx"
)
@ -140,7 +141,7 @@ func (r *artistRepository) toModels(dba []dbArtist) model.Artists {
}
func (r *artistRepository) getIndexKey(a *model.Artist) string {
name := strings.ToLower(utils.NoArticle(a.Name))
name := strings.ToLower(str.NoArticle(a.Name))
for k, v := range r.indexGroups {
key := strings.ToLower(k)
if strings.HasPrefix(name, key) {

6
persistence/sql_search.go
View File

@ -6,11 +6,11 @@ import (
. "github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/str"
)
func getFullText(text ...string) string {
fullText := utils.SanitizeStrings(text...)
fullText := str.SanitizeStrings(text...)
return " " + fullText
}
@ -39,7 +39,7 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{},
}
func fullTextExpr(value string) Sqlizer {
q := utils.SanitizeStrings(value)
q := str.SanitizeStrings(value)
if q == "" {
return nil
}

12
scanner/mapping.go
View File

@ -11,7 +11,7 @@ import (
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/scanner/metadata"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/str"
)
type MediaFileMapper struct {
@ -56,10 +56,10 @@ func (s MediaFileMapper) ToMediaFile(md metadata.Tags) model.MediaFile {
mf.SortAlbumName = md.SortAlbum()
mf.SortArtistName = md.SortArtist()
mf.SortAlbumArtistName = md.SortAlbumArtist()
mf.OrderTitle = utils.SanitizeFieldForSorting(mf.Title)
mf.OrderAlbumName = utils.SanitizeFieldForSortingNoArticle(mf.Album)
mf.OrderArtistName = utils.SanitizeFieldForSortingNoArticle(mf.Artist)
mf.OrderAlbumArtistName = utils.SanitizeFieldForSortingNoArticle(mf.AlbumArtist)
mf.OrderTitle = str.SanitizeFieldForSorting(mf.Title)
mf.OrderAlbumName = str.SanitizeFieldForSortingNoArticle(mf.Album)
mf.OrderArtistName = str.SanitizeFieldForSortingNoArticle(mf.Artist)
mf.OrderAlbumArtistName = str.SanitizeFieldForSortingNoArticle(mf.AlbumArtist)
mf.CatalogNum = md.CatalogNum()
mf.MbzRecordingID = md.MbzRecordingID()
mf.MbzReleaseTrackID = md.MbzReleaseTrackID()
@ -72,7 +72,7 @@ func (s MediaFileMapper) ToMediaFile(md metadata.Tags) model.MediaFile {
mf.RgAlbumPeak = md.RGAlbumPeak()
mf.RgTrackGain = md.RGTrackGain()
mf.RgTrackPeak = md.RGTrackPeak()
mf.Comment = utils.SanitizeText(md.Comment())
mf.Comment = str.SanitizeText(md.Comment())
mf.Lyrics = md.Lyrics()
mf.Bpm = md.Bpm()
mf.CreatedAt = md.BirthTime()

8
server/serve_index.go
View File

@ -15,8 +15,8 @@ import (
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
"github.com/navidrome/navidrome/utils/str"
)
func Index(ds model.DataStore, fs fs.FS) http.HandlerFunc {
@ -42,9 +42,9 @@ func serveIndex(ds model.DataStore, fs fs.FS, shareInfo *model.Share) http.Handl
"version": consts.Version,
"firstTime": firstTime,
"variousArtistsId": consts.VariousArtistsID,
"baseURL": utils.SanitizeText(strings.TrimSuffix(conf.Server.BasePath, "/")),
"loginBackgroundURL": utils.SanitizeText(conf.Server.UILoginBackgroundURL),
"welcomeMessage": utils.SanitizeText(conf.Server.UIWelcomeMessage),
"baseURL": str.SanitizeText(strings.TrimSuffix(conf.Server.BasePath, "/")),
"loginBackgroundURL": str.SanitizeText(conf.Server.UILoginBackgroundURL),
"welcomeMessage": str.SanitizeText(conf.Server.UIWelcomeMessage),
"maxSidebarPlaylists": conf.Server.MaxSidebarPlaylists,
"enableTranscodingConfig": conf.Server.EnableTranscodingConfig,
"enableDownloads": conf.Server.EnableDownloads,

32
utils/sanitize_strings_test.go
View File

@ -1,32 +0,0 @@
package utils
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("SanitizeStrings", func() {
It("returns all lowercase chars", func() {
Expect(SanitizeStrings("Some Text")).To(Equal("some text"))
})
It("removes accents", func() {
Expect(SanitizeStrings("Quintão")).To(Equal("quintao"))
})
It("remove extra spaces", func() {
Expect(SanitizeStrings(" some text ")).To(Equal("some text"))
})
It("remove duplicated words", func() {
Expect(SanitizeStrings("legião urbana urbana legiÃo")).To(Equal("legiao urbana"))
})
It("remove symbols", func() {
Expect(SanitizeStrings("Tom’s Diner ' “40” ‘A’")).To(Equal("40 a diner toms"))
})
It("remove opening brackets", func() {
Expect(SanitizeStrings("[Five Years]")).To(Equal("five years"))
})
})

12
utils/sanitize_strings.go → utils/str/sanitize_strings.go
View File

@ -1,4 +1,4 @@
package utils
package str
import (
"html"
@ -38,3 +38,13 @@ func SanitizeText(text string) string {
s := policy.Sanitize(text)
return html.UnescapeString(s)
}
func SanitizeFieldForSorting(originalValue string) string {
v := strings.TrimSpace(sanitize.Accents(originalValue))
return strings.ToLower(v)
}
func SanitizeFieldForSortingNoArticle(originalValue string) string {
v := strings.TrimSpace(sanitize.Accents(originalValue))
return strings.ToLower(NoArticle(v))
}

66
utils/str/sanitize_strings_test.go Normal file
View File

@ -0,0 +1,66 @@
package str_test
import (
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/utils/str"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Sanitize Strings", func() {
Describe("SanitizeStrings", func() {
It("returns all lowercase chars", func() {
Expect(str.SanitizeStrings("Some Text")).To(Equal("some text"))
})
It("removes accents", func() {
Expect(str.SanitizeStrings("Quintão")).To(Equal("quintao"))
})
It("remove extra spaces", func() {
Expect(str.SanitizeStrings(" some text ")).To(Equal("some text"))
})
It("remove duplicated words", func() {
Expect(str.SanitizeStrings("legião urbana urbana legiÃo")).To(Equal("legiao urbana"))
})
It("remove symbols", func() {
Expect(str.SanitizeStrings("Tom’s Diner ' “40” ‘A’")).To(Equal("40 a diner toms"))
})
It("remove opening brackets", func() {
Expect(str.SanitizeStrings("[Five Years]")).To(Equal("five years"))
})
})
Describe("SanitizeFieldForSorting", func() {
BeforeEach(func() {
conf.Server.IgnoredArticles = "The O"
})
It("sanitize accents", func() {
Expect(str.SanitizeFieldForSorting("Céu")).To(Equal("ceu"))
})
It("removes articles", func() {
Expect(str.SanitizeFieldForSorting("The Beatles")).To(Equal("the beatles"))
})
It("removes accented articles", func() {
Expect(str.SanitizeFieldForSorting("Õ Blésq Blom")).To(Equal("o blesq blom"))
})
})
Describe("SanitizeFieldForSortingNoArticle", func() {
BeforeEach(func() {
conf.Server.IgnoredArticles = "The O"
})
It("sanitize accents", func() {
Expect(str.SanitizeFieldForSortingNoArticle("Céu")).To(Equal("ceu"))
})
It("removes articles", func() {
Expect(str.SanitizeFieldForSortingNoArticle("The Beatles")).To(Equal("beatles"))
})
It("removes accented articles", func() {
Expect(str.SanitizeFieldForSortingNoArticle("Õ Blésq Blom")).To(Equal("blesq blom"))
})
})
})

25
utils/strings.go → utils/str/str.go
View File

@ -1,12 +1,23 @@
package utils
package str
import (
"strings"
"github.com/deluan/sanitize"
"github.com/navidrome/navidrome/conf"
)
func Clear(name string) string {
r := strings.NewReplacer(
"–", "-",
"‐", "-",
"“", `"`,
"”", `"`,
"‘", `'`,
"’", `'`,
)
return r.Replace(name)
}
func NoArticle(name string) string {
articles := strings.Split(conf.Server.IgnoredArticles, " ")
for _, a := range articles {
@ -33,13 +44,3 @@ func LongestCommonPrefix(list []string) string {
}
return list[0]
}
func SanitizeFieldForSorting(originalValue string) string {
v := strings.TrimSpace(sanitize.Accents(originalValue))
return strings.ToLower(v)
}
func SanitizeFieldForSortingNoArticle(originalValue string) string {
v := strings.TrimSpace(sanitize.Accents(originalValue))
return strings.ToLower(NoArticle(v))
}

13
utils/str/str_suite_test.go Normal file
View File

@ -0,0 +1,13 @@
package str_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestStrClear(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Str Suite")
}

56
utils/strings_test.go → utils/str/str_test.go
View File

@ -1,11 +1,24 @@
package utils
package str_test
import (
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/utils/str"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Clean", func() {
DescribeTable("replaces some Unicode chars with their equivalent ASCII",
func(input, expected string) {
Expect(str.Clear(input)).To(Equal(expected))
},
Entry("k-os", "k–os", "k-os"),
Entry("k‐os", "k‐os", "k-os"),
Entry(`"Weird" Al Yankovic`, "“Weird” Al Yankovic", `"Weird" Al Yankovic`),
Entry("Single quotes", "‘Single’ quotes", "'Single' quotes"),
)
})
var _ = Describe("Strings", func() {
Describe("NoArticle", func() {
Context("Empty articles list", func() {
@ -13,10 +26,10 @@ var _ = Describe("Strings", func() {
conf.Server.IgnoredArticles = ""
})
It("returns empty if string is empty", func() {
Expect(NoArticle("")).To(BeEmpty())
Expect(str.NoArticle("")).To(BeEmpty())
})
It("returns same string", func() {
Expect(NoArticle("The Beatles")).To(Equal("The Beatles"))
Expect(str.NoArticle("The Beatles")).To(Equal("The Beatles"))
})
})
Context("Default articles", func() {
@ -24,49 +37,20 @@ var _ = Describe("Strings", func() {
conf.Server.IgnoredArticles = "The El La Los Las Le Les Os As O A"
})
It("returns empty if string is empty", func() {
Expect(NoArticle("")).To(BeEmpty())
Expect(str.NoArticle("")).To(BeEmpty())
})
It("remove prefix article from string", func() {
Expect(NoArticle("Os Paralamas do Sucesso")).To(Equal("Paralamas do Sucesso"))
Expect(str.NoArticle("Os Paralamas do Sucesso")).To(Equal("Paralamas do Sucesso"))
})
It("does not remove article if it is part of the first word", func() {
Expect(NoArticle("Thelonious Monk")).To(Equal("Thelonious Monk"))
Expect(str.NoArticle("Thelonious Monk")).To(Equal("Thelonious Monk"))
})
})
})
Describe("sanitizeFieldForSorting", func() {
BeforeEach(func() {
conf.Server.IgnoredArticles = "The O"
})
It("sanitize accents", func() {
Expect(SanitizeFieldForSorting("Céu")).To(Equal("ceu"))
})
It("removes articles", func() {
Expect(SanitizeFieldForSorting("The Beatles")).To(Equal("the beatles"))
})
It("removes accented articles", func() {
Expect(SanitizeFieldForSorting("Õ Blésq Blom")).To(Equal("o blesq blom"))
})
})
Describe("SanitizeFieldForSortingNoArticle", func() {
BeforeEach(func() {
conf.Server.IgnoredArticles = "The O"
})
It("sanitize accents", func() {
Expect(SanitizeFieldForSortingNoArticle("Céu")).To(Equal("ceu"))
})
It("removes articles", func() {
Expect(SanitizeFieldForSortingNoArticle("The Beatles")).To(Equal("beatles"))
})
It("removes accented articles", func() {
Expect(SanitizeFieldForSortingNoArticle("Õ Blésq Blom")).To(Equal("blesq blom"))
})
})
Describe("LongestCommonPrefix", func() {
It("finds the longest common prefix", func() {
Expect(LongestCommonPrefix(testPaths)).To(Equal("/Music/iTunes 1/iTunes Media/Music/"))
Expect(str.LongestCommonPrefix(testPaths)).To(Equal("/Music/iTunes 1/iTunes Media/Music/"))
})
})