mirrors
/
streamrip
mirror of https://github.com/nathom/streamrip.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Deezer working

This commit is contained in:
Nathan Thomas 2023-12-20 16:55:34 -08:00
parent 349e46739c
commit 64b94bfea5
28 changed files with 245 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
streamrip/client/client.py
View File

@ -52,7 +52,7 @@ class Client(ABC):
if headers is None:
headers = {}
return aiohttp.ClientSession(
headers={"User-Agent": DEFAULT_USER_AGENT}, **headers
headers={"User-Agent": DEFAULT_USER_AGENT}, **headers,
)
def __del__(self):

92
streamrip/client/deezer.py
View File

@ -1,6 +1,6 @@
import asyncio
import binascii
import hashlib
import json
import logging
import deezer
@ -12,6 +12,7 @@ from .client import Client
from .downloadable import DeezerDownloadable
logger = logging.getLogger("streamrip")
logging.captureWarnings(True)
class DeezerClient(Client):
@ -37,30 +38,62 @@ class DeezerClient(Client):
async def get_metadata(self, item_id: str, media_type: str) -> dict:
# TODO: open asyncio PR to deezer py and integrate
request_functions = {
"track": self.client.api.get_track,
"album": self.client.api.get_album,
"playlist": self.client.api.get_playlist,
"artist": self.client.api.get_artist,
}
get_item = request_functions[media_type]
item = get_item(item_id)
if media_type in ("album", "playlist"):
tracks = getattr(self.client.api, f"get_{media_type}_tracks")(
item_id, limit=-1
)
item["tracks"] = tracks["data"]
item["track_total"] = len(tracks["data"])
if media_type == "track":
return await self.get_track(item_id)
elif media_type == "album":
return await self.get_album(item_id)
elif media_type == "playlist":
return await self.get_playlist(item_id)
elif media_type == "artist":
albums = self.client.api.get_artist_albums(item_id)
item["albums"] = albums["data"]
elif media_type == "track":
# Because they give incomplete information about the album
# we need to make another request
item["album"] = await self.get_metadata(item["album"]["id"], "album")
return await self.get_artist(item_id)
else:
raise Exception(f"Media type {media_type} not available on deezer")
async def get_track(self, item_id: str) -> dict:
item = await asyncio.to_thread(self.client.api.get_track, item_id)
album_id = item["album"]["id"]
try:
album_metadata, album_tracks = await asyncio.gather(
asyncio.to_thread(self.client.api.get_album, album_id),
asyncio.to_thread(self.client.api.get_album_tracks, album_id),
)
except Exception as e:
logger.error("Got exception from deezer API %s", e)
# item["album"] = {"readable": False, "tracks": [], "track_total": 0}
return item
album_metadata["tracks"] = album_tracks["data"]
album_metadata["track_total"] = len(album_tracks["data"])
item["album"] = album_metadata
return item
async def get_album(self, item_id: str) -> dict:
album_metadata, album_tracks = await asyncio.gather(
asyncio.to_thread(self.client.api.get_album, item_id),
asyncio.to_thread(self.client.api.get_album_tracks, item_id),
)
album_metadata["tracks"] = album_tracks["data"]
album_metadata["track_total"] = len(album_tracks["data"])
return album_metadata
async def get_playlist(self, item_id: str) -> dict:
pl_metadata, pl_tracks = await asyncio.gather(
asyncio.to_thread(self.client.api.get_playlist, item_id),
asyncio.to_thread(self.client.api.get_playlist_tracks, item_id),
)
pl_metadata["tracks"] = pl_tracks["data"]
pl_metadata["track_total"] = len(pl_tracks["data"])
return pl_metadata
async def get_artist(self, item_id: str) -> dict:
artist, albums = await asyncio.gather(
asyncio.to_thread(self.client.api.get_artist, item_id),
asyncio.to_thread(self.client.api.get_artist_albums, item_id),
)
artist["albums"] = albums["data"]
return artist
async def search(self, media_type: str, query: str, limit: int = 200):
# TODO: use limit parameter
if media_type == "featured":
@ -81,7 +114,7 @@ class DeezerClient(Client):
return response
async def get_downloadable(
self, item_id: str, quality: int = 2
self, item_id: str, quality: int = 2,
) -> DeezerDownloadable:
# TODO: optimize such that all of the ids are requested at once
dl_info: dict = {"quality": quality, "id": item_id}
@ -120,28 +153,29 @@ class DeezerClient(Client):
token = track_info["TRACK_TOKEN"]
try:
logger.debug("Fetching deezer url with token %s", token)
url = self.client.get_track_url(token, format_str)
except deezer.WrongLicense:
raise NonStreamable(
"The requested quality is not available with your subscription. "
"Deezer HiFi is required for quality 2. Otherwise, the maximum "
"quality allowed is 1."
"quality allowed is 1.",
)
except deezer.WrongGeolocation:
raise NonStreamable(
"The requested track is not available. This may be due to your country/location."
"The requested track is not available. This may be due to your country/location.",
)
if url is None:
url = self._get_encrypted_file_url(
item_id, track_info["MD5_ORIGIN"], track_info["MEDIA_VERSION"]
item_id, track_info["MD5_ORIGIN"], track_info["MEDIA_VERSION"],
)
dl_info["url"] = url
return DeezerDownloadable(self.session, dl_info)
def _get_encrypted_file_url(
self, meta_id: str, track_hash: str, media_version: str
self, meta_id: str, track_hash: str, media_version: str,
):
logger.debug("Unable to fetch URL. Trying encryption method.")
format_number = 1
@ -152,7 +186,7 @@ class DeezerClient(Client):
str(format_number).encode(),
str(meta_id).encode(),
str(media_version).encode(),
)
),
)
url_hash = hashlib.md5(url_bytes).hexdigest()
info_bytes = bytearray(url_hash.encode())
@ -164,7 +198,7 @@ class DeezerClient(Client):
info_bytes.extend(b"." * padding_len)
path = binascii.hexlify(
AES.new("jo6aey6haid2Teih".encode(), AES.MODE_ECB).encrypt(info_bytes)
AES.new(b"jo6aey6haid2Teih", AES.MODE_ECB).encrypt(info_bytes),
).decode("utf-8")
return f"https://e-cdns-proxy-{track_hash[0]}.dzcdn.net/mobile/1/{path}"

23
streamrip/client/downloadable.py
View File

@ -26,7 +26,7 @@ logger = logging.getLogger("streamrip")
def generate_temp_path(url: str):
return os.path.join(
tempfile.gettempdir(), f"__streamrip_{hash(url)}_{time.time()}.download"
tempfile.gettempdir(), f"__streamrip_{hash(url)}_{time.time()}.download",
)
@ -39,9 +39,7 @@ class Downloadable(ABC):
_size: Optional[int] = None
async def download(self, path: str, callback: Callable[[int], Any]):
tmp = generate_temp_path(self.url)
await self._download(tmp, callback)
shutil.move(tmp, path)
await self._download(path, callback)
async def size(self) -> int:
if self._size is not None:
@ -55,7 +53,7 @@ class Downloadable(ABC):
@abstractmethod
async def _download(self, path: str, callback: Callable[[int], None]):
raise NotImplemented
raise NotImplementedError
class BasicDownloadable(Downloadable):
@ -92,7 +90,7 @@ class DeezerDownloadable(Downloadable):
self.extension = "mp3"
else:
self.extension = "flac"
self.id = info["id"]
self.id = str(info["id"])
async def _download(self, path: str, callback):
# with requests.Session().get(self.url, allow_redirects=True) as resp:
@ -121,7 +119,7 @@ class DeezerDownloadable(Downloadable):
else:
blowfish_key = self._generate_blowfish_key(self.id)
logger.debug(
f"Deezer file (id %s) at %s is encrypted. Decrypting with %s",
"Deezer file (id %s) at %s is encrypted. Decrypting with %s",
self.id,
self.url,
blowfish_key,
@ -182,7 +180,8 @@ class DeezerDownloadable(Downloadable):
class TidalDownloadable(Downloadable):
"""A wrapper around BasicDownloadable that includes Tidal-specific
error messages."""
error messages.
"""
def __init__(self, session: aiohttp.ClientSession, info: dict):
self.session = session
@ -192,7 +191,7 @@ class TidalDownloadable(Downloadable):
# Turn CamelCase code into a readable sentence
words = re.findall(r"([A-Z][a-z]+)", restrictions[0]["code"])
raise NonStreamable(
words[0] + " " + " ".join(map(str.lower, words[1:]))
words[0] + " " + " ".join(map(str.lower, words[1:])),
)
raise NonStreamable(f"Tidal download: dl_info = {info}")
@ -229,6 +228,7 @@ class SoundcloudDownloadable(Downloadable):
await engine.convert(path)
async def _download_mp3(self, path: str, callback):
# TODO: make progress bar reflect bytes
async with self.session.get(self.url) as resp:
content = await resp.text("utf-8")
@ -270,7 +270,6 @@ async def concat_audio_files(paths: list[str], out: str, ext: str, max_files_ope
Recurses log_{max_file_open}(len(paths)) times.
"""
if shutil.which("ffmpeg") is None:
raise Exception("FFmpeg must be installed.")
@ -286,7 +285,7 @@ async def concat_audio_files(paths: list[str], out: str, ext: str, max_files_ope
tempdir = tempfile.gettempdir()
outpaths = [
os.path.join(
tempdir, f"__streamrip_ffmpeg_{hash(paths[i*max_files_open])}.{ext}"
tempdir, f"__streamrip_ffmpeg_{hash(paths[i*max_files_open])}.{ext}",
)
for i in range(num_batches)
]
@ -320,7 +319,7 @@ async def concat_audio_files(paths: list[str], out: str, ext: str, max_files_ope
for proc in processes:
if proc.returncode != 0:
raise Exception(
f"FFMPEG returned with status code {proc.returncode} error: {proc.stderr} output: {proc.stdout}"
f"FFMPEG returned with status code {proc.returncode} error: {proc.stderr} output: {proc.stdout}",
)
# Recurse on remaining batches

21
streamrip/client/qobuz.py
View File

@ -104,7 +104,7 @@ class QobuzSpoofer:
secrets.move_to_end(keypairs[1][0], last=False)
info_extras_regex = self.info_extras_regex.format(
timezones="|".join(timezone.capitalize() for timezone in secrets)
timezones="|".join(timezone.capitalize() for timezone in secrets),
)
info_extras_matches = re.finditer(info_extras_regex, self.bundle)
for match in info_extras_matches:
@ -113,7 +113,7 @@ class QobuzSpoofer:
for secret_pair in secrets:
secrets[secret_pair] = base64.standard_b64decode(
"".join(secrets[secret_pair])[:-44]
"".join(secrets[secret_pair])[:-44],
).decode("utf-8")
vals: List[str] = list(secrets.values())
@ -141,7 +141,7 @@ class QobuzClient(Client):
self.logged_in = False
self.config = config
self.rate_limiter = self.get_rate_limiter(
config.session.downloads.requests_per_minute
config.session.downloads.requests_per_minute,
)
self.secret: Optional[str] = None
@ -227,7 +227,7 @@ class QobuzClient(Client):
if status != 200:
raise NonStreamable(
f'Error fetching metadata. Message: "{resp["message"]}"'
f'Error fetching metadata. Message: "{resp["message"]}"',
)
return resp
@ -275,16 +275,16 @@ class QobuzClient(Client):
# Turn CamelCase code into a readable sentence
words = re.findall(r"([A-Z][a-z]+)", restrictions[0]["code"])
raise NonStreamable(
words[0] + " " + " ".join(map(str.lower, words[1:])) + "."
words[0] + " " + " ".join(map(str.lower, words[1:])) + ".",
)
raise NonStreamable
return BasicDownloadable(
self.session, stream_url, "flac" if quality > 1 else "mp3"
self.session, stream_url, "flac" if quality > 1 else "mp3",
)
async def _paginate(
self, epoint: str, params: dict, limit: Optional[int] = None
self, epoint: str, params: dict, limit: Optional[int] = None,
) -> list[dict]:
"""Paginate search results.
@ -292,7 +292,8 @@ class QobuzClient(Client):
limit: If None, all the results are yielded. Otherwise a maximum
of `limit` results are yielded.
returns:
Returns
-------
Generator that yields (status code, response) tuples
"""
params.update({"limit": limit or 500})
@ -339,7 +340,7 @@ class QobuzClient(Client):
async def _get_valid_secret(self, secrets: list[str]) -> str:
results = await asyncio.gather(
*[self._test_secret(secret) for secret in secrets]
*[self._test_secret(secret) for secret in secrets],
)
working_secrets = [r for r in results if r is not None]
@ -358,7 +359,7 @@ class QobuzClient(Client):
return None
async def _request_file_url(
self, track_id: str, quality: int, secret: str
self, track_id: str, quality: int, secret: str,
) -> tuple[int, dict]:
quality = self.get_quality(quality)
unix_ts = time.time()

14
streamrip/client/soundcloud.py
View File

@ -26,7 +26,7 @@ class SoundcloudClient(Client):
self.global_config = config
self.config = config.session.soundcloud
self.rate_limiter = self.get_rate_limiter(
config.session.downloads.requests_per_minute
config.session.downloads.requests_per_minute,
)
async def login(self):
@ -50,10 +50,12 @@ class SoundcloudClient(Client):
"""Fetch metadata for an item in Soundcloud API.
Args:
----
item_id (str): Plain soundcloud item ID (e.g 1633786176)
media_type (str): track or playlist
Returns:
-------
API response.
"""
if media_type == "track":
@ -157,7 +159,7 @@ class SoundcloudClient(Client):
resp_json, status = await self._api_request(f"tracks/{item_id}/download")
assert status == 200
return SoundcloudDownloadable(
self.session, {"url": resp_json["redirectUri"], "type": "original"}
self.session, {"url": resp_json["redirectUri"], "type": "original"},
)
if download_info == self.NOT_RESOLVED:
@ -166,11 +168,11 @@ class SoundcloudClient(Client):
# download_info contains mp3 stream url
resp_json, status = await self._request(download_info)
return SoundcloudDownloadable(
self.session, {"url": resp_json["url"], "type": "mp3"}
self.session, {"url": resp_json["url"], "type": "mp3"},
)
async def search(
self, media_type: str, query: str, limit: int = 50, offset: int = 0
self, media_type: str, query: str, limit: int = 50, offset: int = 0,
) -> list[dict]:
# TODO: implement pagination
assert media_type in ("track", "playlist")
@ -234,7 +236,7 @@ class SoundcloudClient(Client):
page_text = await resp.text(encoding="utf-8")
*_, client_id_url_match = re.finditer(
r"<script\s+crossorigin\s+src=\"([^\"]+)\"", page_text
r"<script\s+crossorigin\s+src=\"([^\"]+)\"", page_text,
)
if client_id_url_match is None:
@ -243,7 +245,7 @@ class SoundcloudClient(Client):
client_id_url = client_id_url_match.group(1)
app_version_match = re.search(
r'<script>window\.__sc_version="(\d+)"</script>', page_text
r'<script>window\.__sc_version="(\d+)"</script>', page_text,
)
if app_version_match is None:
raise Exception("Could not find app version in %s" % client_id_url_match)

8
streamrip/client/tidal.py
View File

@ -9,7 +9,7 @@ AUTH_URL = "https://auth.tidal.com/v1/oauth2"
CLIENT_ID = base64.b64decode("elU0WEhWVmtjMnREUG80dA==").decode("iso-8859-1")
CLIENT_SECRET = base64.b64decode(
"VkpLaERGcUpQcXZzUFZOQlY2dWtYVEptd2x2YnR0UDd3bE1scmM3MnNlND0="
"VkpLaERGcUpQcXZzUFZOQlY2dWtYVEptd2x2YnR0UDd3bE1scmM3MnNlND0=",
).decode("iso-8859-1")
@ -24,7 +24,7 @@ class TidalClient(Client):
self.global_config = config
self.config = config.session.tidal
self.rate_limiter = self.get_rate_limiter(
config.session.downloads.requests_per_minute
config.session.downloads.requests_per_minute,
)
async def login(self):
@ -53,7 +53,7 @@ class TidalClient(Client):
"""
headers = {"authorization": f"Bearer {token}"} # temporary
async with self.session.get(
"https://api.tidal.com/v1/sessions", headers=headers
"https://api.tidal.com/v1/sessions", headers=headers,
) as _resp:
resp = await _resp.json()
@ -85,7 +85,7 @@ class TidalClient(Client):
def _update_authorization_from_config(self):
self.session.headers.update(
{"authorization": f"Bearer {self.config.access_token}"}
{"authorization": f"Bearer {self.config.access_token}"},
)
async def _get_auth_status(self, device_code) -> tuple[int, dict[str, int | str]]:

6
streamrip/config.py
View File

@ -220,7 +220,7 @@ DEFAULT_DOWNLOADS_FOLDER = os.path.join(HOME, "StreamripDownloads")
DEFAULT_DOWNLOADS_DB_PATH = os.path.join(APP_DIR, "downloads.db")
DEFAULT_FAILED_DOWNLOADS_DB_PATH = os.path.join(APP_DIR, "failed_downloads.db")
DEFAULT_YOUTUBE_VIDEO_DOWNLOADS_FOLDER = os.path.join(
DEFAULT_DOWNLOADS_FOLDER, "YouTubeVideos"
DEFAULT_DOWNLOADS_FOLDER, "YouTubeVideos",
)
BLANK_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "config.toml")
assert os.path.isfile(BLANK_CONFIG_PATH), "Template config not found"
@ -257,7 +257,7 @@ class ConfigData:
toml = parse(toml_str)
if (v := toml["misc"]["version"]) != CURRENT_CONFIG_VERSION: # type: ignore
raise Exception(
f"Need to update config from {v} to {CURRENT_CONFIG_VERSION}"
f"Need to update config from {v} to {CURRENT_CONFIG_VERSION}",
)
downloads = DownloadsConfig(**toml["downloads"]) # type: ignore
@ -324,7 +324,7 @@ class ConfigData:
update_toml_section_from_config(self.toml["conversion"], self.conversion)
def get_source(
self, source: str
self, source: str,
) -> QobuzConfig | DeezerConfig | SoundcloudConfig | TidalConfig:
d = {
"qobuz": self.qobuz,

9
streamrip/converter.py
View File

@ -48,10 +48,9 @@ class Converter:
:param remove_source: Remove the source file after conversion.
:type remove_source: bool
"""
if shutil.which("ffmpeg") is None:
raise Exception(
"Could not find FFMPEG executable. Install it to convert audio files."
"Could not find FFMPEG executable. Install it to convert audio files.",
)
self.filename = filename
@ -86,7 +85,7 @@ class Converter:
logger.debug("Generated conversion command: %s", self.command)
process = await asyncio.create_subprocess_exec(
*self.command, stderr=asyncio.subprocess.PIPE
*self.command, stderr=asyncio.subprocess.PIPE,
)
out, err = await process.communicate()
if process.returncode == 0 and os.path.isfile(self.tempfile):
@ -129,7 +128,7 @@ class Converter:
elif self.sampling_rate is not None:
raise TypeError(
f"Sampling rate must be int, not {type(self.sampling_rate)}"
f"Sampling rate must be int, not {type(self.sampling_rate)}",
)
if isinstance(self.bit_depth, int):
@ -154,7 +153,7 @@ class Converter:
if self.ffmpeg_arg is not None and self.lossless:
logger.debug(
"Lossless codecs don't support extra arguments; "
"the extra argument will be ignored"
"the extra argument will be ignored",
)
self.ffmpeg_arg = self.default_ffmpeg_arg
return

3
streamrip/db.py
View File

@ -93,7 +93,6 @@ class DatabaseBase(DatabaseInterface):
:param items: a dict of column-name + expected value
:rtype: bool
"""
allowed_keys = set(self.structure.keys())
assert all(
key in allowed_keys for key in items.keys()
@ -115,7 +114,6 @@ class DatabaseBase(DatabaseInterface):
:param items: Column-name + value. Values must be provided for all cols.
:type items: Tuple[str]
"""
assert len(items) == len(self.structure)
params = ", ".join(self.structure.keys())
@ -139,7 +137,6 @@ class DatabaseBase(DatabaseInterface):
:param items:
"""
conditions = " AND ".join(f"{key}=?" for key in items.keys())
command = f"DELETE FROM {self.name} WHERE {conditions}"

2
streamrip/exceptions.py
View File

@ -65,7 +65,7 @@ class NonStreamable(Exception):
(
style("Message:", fg="yellow"),
style(self.message, fg="red"),
)
),
)
return " ".join(base_msg)

2
streamrip/media/album.py
View File

@ -56,7 +56,7 @@ class PendingAlbum(Pending):
meta = AlbumMetadata.from_album_resp(resp, self.client.source)
except NonStreamable:
logger.error(
f"Album {self.id} not available to stream on {self.client.source}"
f"Album {self.id} not available to stream on {self.client.source}",
)
return None

13
streamrip/media/artwork.py
View File

@ -42,12 +42,15 @@ async def download_artwork(
Hi-res (saved) artworks are kept in `folder` as "cover.jpg".
Args:
----
session (aiohttp.ClientSession):
folder (str):
covers (Covers):
config (ArtworkConfig):
for_playlist (bool): Set to disable saved hires covers.
Returns:
-------
(path to embedded artwork, path to hires artwork)
"""
save_artwork, embed = config.save_artwork, config.embed
@ -66,8 +69,8 @@ async def download_artwork(
assert l_url is not None
downloadables.append(
BasicDownloadable(session, l_url, "jpg").download(
saved_cover_path, lambda _: None
)
saved_cover_path, lambda _: None,
),
)
_, embed_url, embed_cover_path = covers.get_size(config.embed_size)
@ -79,8 +82,8 @@ async def download_artwork(
embed_cover_path = os.path.join(embed_dir, f"cover{hash(embed_url)}.jpg")
downloadables.append(
BasicDownloadable(session, embed_url, "jpg").download(
embed_cover_path, lambda _: None
)
embed_cover_path, lambda _: None,
),
)
if len(downloadables) == 0:
@ -108,10 +111,12 @@ def downscale_image(input_image_path: str, max_dimension: int):
"""Downscale an image in place given a maximum allowed dimension.
Args:
----
input_image_path (str): Path to image
max_dimension (int): Maximum dimension allowed
Returns:
-------
"""

8
streamrip/media/media.py
View File

@ -10,17 +10,17 @@ class Media(ABC):
@abstractmethod
async def preprocess(self):
"""Create directories, download cover art, etc."""
raise NotImplemented
raise NotImplementedError
@abstractmethod
async def download(self):
"""Download and tag the actual audio files in the correct directories."""
raise NotImplemented
raise NotImplementedError
@abstractmethod
async def postprocess(self):
"""Update database, run conversion, delete garbage files etc."""
raise NotImplemented
raise NotImplementedError
class Pending(ABC):
@ -29,4 +29,4 @@ class Pending(ABC):
@abstractmethod
async def resolve(self) -> Media | None:
"""Fetch metadata and resolve into a downloadable `Media` object."""
raise NotImplemented
raise NotImplementedError

41
streamrip/media/playlist.py
View File

@ -15,6 +15,7 @@ from ..client import Client
from ..config import Config
from ..console import console
from ..db import Database
from ..exceptions import NonStreamable
from ..filepath_utils import clean_filename
from ..metadata import (
AlbumMetadata,
@ -47,10 +48,16 @@ class PendingPlaylistTrack(Pending):
resp = await self.client.get_metadata(self.id, "track")
album = AlbumMetadata.from_track_resp(resp, self.client.source)
if album is None:
logger.error(
f"Track ({self.id}) not available for stream on {self.client.source}",
)
self.db.set_failed(self.client.source, "track", self.id)
return None
meta = TrackMetadata.from_resp(album, self.client.source, resp)
if meta is None:
logger.error(
f"Track ({self.id}) not available for stream on {self.client.source}"
f"Track ({self.id}) not available for stream on {self.client.source}",
)
self.db.set_failed(self.client.source, "track", self.id)
return None
@ -62,12 +69,18 @@ class PendingPlaylistTrack(Pending):
album.album = self.playlist_name
quality = self.config.session.get_source(self.client.source).quality
embedded_cover_path, downloadable = await asyncio.gather(
self._download_cover(album.covers, self.folder),
self.client.get_downloadable(self.id, quality),
)
try:
embedded_cover_path, downloadable = await asyncio.gather(
self._download_cover(album.covers, self.folder),
self.client.get_downloadable(self.id, quality),
)
except NonStreamable as e:
logger.error("Error fetching download info for track: %s", e)
self.db.set_failed(self.client.source, "track", self.id)
return None
return Track(
meta, downloadable, self.config, self.folder, embedded_cover_path, self.db
meta, downloadable, self.config, self.folder, embedded_cover_path, self.db,
)
async def _download_cover(self, covers: Covers, folder: str) -> str | None:
@ -132,7 +145,7 @@ class PendingPlaylist(Pending):
folder = os.path.join(parent, clean_filename(name))
tracks = [
PendingPlaylistTrack(
id, self.client, self.config, folder, name, position + 1, self.db
id, self.client, self.config, folder, name, position + 1, self.db,
)
for position, id in enumerate(meta.ids())
]
@ -167,7 +180,7 @@ class PendingLastfmPlaylist(Pending):
async def resolve(self) -> Playlist | None:
try:
playlist_title, titles_artists = await self._parse_lastfm_playlist(
self.lastfm_url
self.lastfm_url,
)
except Exception as e:
logger.error("Error occured while parsing last.fm page: %s", e)
@ -212,13 +225,13 @@ class PendingLastfmPlaylist(Pending):
playlist_title,
pos,
self.db,
)
),
)
return Playlist(playlist_title, self.config, self.client, pending_tracks)
async def _make_query(
self, query: str, s: Status, callback
self, query: str, s: Status, callback,
) -> tuple[str | None, bool]:
"""Try searching for `query` with main source. If that fails, try with next source.
@ -248,7 +261,7 @@ class PendingLastfmPlaylist(Pending):
s.found += 1
return (
SearchResults.from_pages(
self.fallback_client.source, "track", pages
self.fallback_client.source, "track", pages,
)
.results[0]
.id
@ -259,7 +272,7 @@ class PendingLastfmPlaylist(Pending):
return None, True
async def _parse_lastfm_playlist(
self, playlist_url: str
self, playlist_url: str,
) -> tuple[str, list[tuple[str, str]]]:
"""From a last.fm url, return the playlist title, and a list of
track titles and artist names.
@ -276,7 +289,7 @@ class PendingLastfmPlaylist(Pending):
title_tags = re.compile(r'<a\s+href="[^"]+"\s+title="([^"]+)"')
re_total_tracks = re.compile(r'data-playlisting-entry-count="(\d+)"')
re_playlist_title_match = re.compile(
r'<h1 class="playlisting-playlist-header-title">([^<]+)</h1>'
r'<h1 class="playlisting-playlist-header-title">([^<]+)</h1>',
)
def find_title_artist_pairs(page_text):
@ -324,7 +337,7 @@ class PendingLastfmPlaylist(Pending):
return playlist_title, title_artist_pairs
async def _make_query_mock(
self, _: str, s: Status, callback
self, _: str, s: Status, callback,
) -> tuple[str | None, bool]:
await asyncio.sleep(random.uniform(1, 20))
if random.randint(0, 4) >= 1:

23
streamrip/media/track.py
View File

@ -73,13 +73,13 @@ class Track(Media):
c = self.config.session.filepaths
formatter = c.track_format
track_path = clean_filename(
self.meta.format_track_path(formatter), restrict=c.restrict_characters
self.meta.format_track_path(formatter), restrict=c.restrict_characters,
)
if c.truncate_to > 0 and len(track_path) > c.truncate_to:
track_path = track_path[: c.truncate_to]
self.download_path = os.path.join(
self.folder, f"{track_path}.{self.downloadable.extension}"
self.folder, f"{track_path}.{self.downloadable.extension}",
)
@ -97,7 +97,7 @@ class PendingTrack(Pending):
async def resolve(self) -> Track | None:
if self.db.downloaded(self.id):
logger.info(
f"Skipping track {self.id}. Marked as downloaded in the database."
f"Skipping track {self.id}. Marked as downloaded in the database.",
)
return None
@ -112,7 +112,7 @@ class PendingTrack(Pending):
quality = self.config.session.get_source(source).quality
downloadable = await self.client.get_downloadable(self.id, quality)
return Track(
meta, downloadable, self.config, self.folder, self.cover_path, self.db
meta, downloadable, self.config, self.folder, self.cover_path, self.db,
)
@ -132,7 +132,7 @@ class PendingSingle(Pending):
async def resolve(self) -> Track | None:
if self.db.downloaded(self.id):
logger.info(
f"Skipping track {self.id}. Marked as downloaded in the database."
f"Skipping track {self.id}. Marked as downloaded in the database.",
)
return None
@ -144,17 +144,26 @@ class PendingSingle(Pending):
# Patch for soundcloud
# self.id = resp["id"]
album = AlbumMetadata.from_track_resp(resp, self.client.source)
if album is None:
self.db.set_failed(self.client.source, "track", self.id)
logger.error(
f"Cannot stream track (am) ({self.id}) on {self.client.source}",
)
return None
meta = TrackMetadata.from_resp(album, self.client.source, resp)
if meta is None:
self.db.set_failed(self.client.source, "track", self.id)
logger.error(f"Cannot stream track ({self.id}) on {self.client.source}")
logger.error(
f"Cannot stream track (tm) ({self.id}) on {self.client.source}",
)
return None
quality = getattr(self.config.session, self.client.source).quality
assert isinstance(quality, int)
folder = os.path.join(
self.config.session.downloads.folder, self._format_folder(album)
self.config.session.downloads.folder, self._format_folder(album),
)
os.makedirs(folder, exist_ok=True)

11
streamrip/metadata/album_metadata.py
View File

@ -1,6 +1,5 @@
from __future__ import annotations
import json
import logging
import re
from dataclasses import dataclass
@ -162,7 +161,7 @@ class AlbumMetadata:
)
@classmethod
def from_deezer(cls, resp: dict) -> AlbumMetadata:
def from_deezer(cls, resp: dict) -> AlbumMetadata | None:
album = resp.get("title", "Unknown Album")
tracktotal = typed(resp.get("track_total", 0) or resp.get("nb_tracks", 0), int)
disctotal = typed(resp["tracks"][-1]["disk_number"], int)
@ -228,7 +227,7 @@ class AlbumMetadata:
track_id = track["id"]
bit_depth, sampling_rate = None, None
explicit = typed(
safe_get(track, "publisher_metadata", "explicit", default=False), bool
safe_get(track, "publisher_metadata", "explicit", default=False), bool,
)
genre = typed(track["genre"], str)
artist = typed(safe_get(track, "publisher_metadata", "artist"), str | None)
@ -239,7 +238,7 @@ class AlbumMetadata:
label = typed(track["label_name"], str | None)
description = typed(track.get("description"), str | None)
album_title = typed(
safe_get(track, "publisher_metadata", "album_title"), str | None
safe_get(track, "publisher_metadata", "album_title"), str | None,
)
album_title = album_title or "Unknown album"
copyright = typed(safe_get(track, "publisher_metadata", "p_line"), str | None)
@ -285,7 +284,7 @@ class AlbumMetadata:
raise NotImplementedError
@classmethod
def from_track_resp(cls, resp: dict, source: str) -> AlbumMetadata:
def from_track_resp(cls, resp: dict, source: str) -> AlbumMetadata | None:
if source == "qobuz":
return cls.from_qobuz(resp["album"])
if source == "tidal":
@ -297,7 +296,7 @@ class AlbumMetadata:
raise Exception("Invalid source")
@classmethod
def from_album_resp(cls, resp: dict, source: str) -> AlbumMetadata:
def from_album_resp(cls, resp: dict, source: str) -> AlbumMetadata | None:
if source == "qobuz":
return cls.from_qobuz(resp)
if source == "tidal":

4
streamrip/metadata/covers.py
View File

@ -78,7 +78,7 @@ class Covers:
def from_soundcloud(cls, resp):
c = cls()
cover_url = (resp["artwork_url"] or resp["user"].get("avatar_url")).replace(
"large", "t500x500"
"large", "t500x500",
)
c.set_cover_url("large", cover_url)
return c
@ -118,7 +118,7 @@ class Covers:
possibles = (80, 160, 320, 640, 1280)
assert size in possibles, f"size must be in {possibles}"
return TIDAL_COVER_URL.format(
uuid=uuid.replace("-", "/"), height=size, width=size
uuid=uuid.replace("-", "/"), height=size, width=size,
)
def __repr__(self):

23
streamrip/metadata/playlist_metadata.py
View File

@ -43,7 +43,7 @@ def parse_soundcloud_id(item_id: str) -> tuple[str, str]:
@dataclass(slots=True)
class PlaylistMetadata:
name: str
tracks: list[TrackMetadata]
tracks: list[TrackMetadata] | list[str]
@classmethod
def from_qobuz(cls, resp: dict):
@ -53,7 +53,7 @@ class PlaylistMetadata:
for i, track in enumerate(resp["tracks"]["items"]):
meta = TrackMetadata.from_qobuz(
AlbumMetadata.from_qobuz(track["album"]), track
AlbumMetadata.from_qobuz(track["album"]), track,
)
if meta is None:
logger.error(f"Track {i+1} in playlist {name} not available for stream")
@ -67,6 +67,7 @@ class PlaylistMetadata:
"""Convert a (modified) soundcloud API response to PlaylistMetadata.
Args:
----
resp (dict): The response, except there should not be any partially resolved items
in the playlist.
@ -74,6 +75,7 @@ class PlaylistMetadata:
elements in resp['tracks'] should be replaced with their full metadata.
Returns:
-------
PlaylistMetadata object.
"""
name = typed(resp["title"], str)
@ -83,8 +85,19 @@ class PlaylistMetadata:
]
return cls(name, tracks)
def ids(self):
return [track.info.id for track in self.tracks]
@classmethod
def from_deezer(cls, resp: dict):
name = typed(resp["title"], str)
tracks = [str(track["id"]) for track in resp["tracks"]]
return cls(name, tracks)
def ids(self) -> list[str]:
if len(self.tracks) == 0:
return []
if isinstance(self.tracks[0], str):
return self.tracks # type: ignore
return [track.info.id for track in self.tracks] # type: ignore
@classmethod
def from_resp(cls, resp: dict, source: str):
@ -92,5 +105,7 @@ class PlaylistMetadata:
return cls.from_qobuz(resp)
elif source == "soundcloud":
return cls.from_soundcloud(resp)
elif source == "deezer":
return cls.from_deezer(resp)
else:
raise NotImplementedError(source)

4
streamrip/metadata/search_results.py
View File

@ -135,7 +135,7 @@ class AlbumSummary(Summary):
or "Unknown"
)
num_tracks = item.get("tracks_count", 0) or len(
item.get("tracks", []) or item.get("items", [])
item.get("tracks", []) or item.get("items", []),
)
date_released = (
@ -184,7 +184,7 @@ class PlaylistSummary(Summary):
def preview(self) -> str:
wrapped = "\n".join(
textwrap.wrap(self.description, os.get_terminal_size().columns - 4 or 70)
textwrap.wrap(self.description, os.get_terminal_size().columns - 4 or 70),
)
return f"{self.num_tracks} tracks\n\nDescription:\n{wrapped}\n\nid:{self.id}"

8
streamrip/metadata/tagger.py
View File

@ -3,10 +3,12 @@ import os
from enum import Enum
import aiofiles
import mutagen.id3 as id3
from mutagen import id3
from mutagen.flac import FLAC, Picture
from mutagen.id3 import APIC # type: ignore
from mutagen.id3 import ID3
from mutagen.id3 import (
APIC, # type: ignore
ID3,
)
from mutagen.mp4 import MP4, MP4Cover
from .track_metadata import TrackMetadata

15
streamrip/metadata/track_metadata.py
View File

@ -1,12 +1,15 @@
from __future__ import annotations
import json
import logging
from dataclasses import dataclass
from typing import Optional
from ..exceptions import NonStreamable
from .album_metadata import AlbumMetadata
from .util import safe_get, typed
logger = logging.getLogger("streamrip")
@dataclass(slots=True)
class TrackInfo:
@ -81,7 +84,11 @@ class TrackMetadata:
)
@classmethod
def from_deezer(cls, album: AlbumMetadata, resp) -> TrackMetadata:
def from_deezer(cls, album: AlbumMetadata, resp) -> TrackMetadata | None:
with open("resp.json", "w") as f:
json.dump(resp, f)
logger.debug(resp.keys())
track_id = str(resp["id"])
bit_depth = 16
sampling_rate = 44.1
@ -116,7 +123,7 @@ class TrackMetadata:
track_id = track["id"]
bit_depth, sampling_rate = None, None
explicit = typed(
safe_get(track, "publisher_metadata", "explicit", default=False), bool
safe_get(track, "publisher_metadata", "explicit", default=False), bool,
)
title = typed(track["title"].strip(), str)
@ -143,7 +150,7 @@ class TrackMetadata:
@classmethod
def from_tidal(cls, album: AlbumMetadata, resp) -> TrackMetadata:
raise NotImplemented
raise NotImplementedError
@classmethod
def from_resp(cls, album: AlbumMetadata, source, resp) -> TrackMetadata | None:

2
streamrip/metadata/util.py
View File

@ -26,7 +26,7 @@ def typed(thing, expected_type: Type[T]) -> T:
def get_quality_id(
bit_depth: Optional[int], sampling_rate: Optional[int | float]
bit_depth: Optional[int], sampling_rate: Optional[int | float],
) -> int:
"""Get the universal quality id from bit depth and sampling rate.

1
streamrip/progress.py
View File

@ -5,7 +5,6 @@ from rich.console import Group
from rich.live import Live
from rich.progress import (
BarColumn,
DownloadColumn,
Progress,
TextColumn,
TimeRemainingColumn,

29
streamrip/rip/cli.py
View File

@ -33,7 +33,7 @@ def coro(f):
)
@click.version_option(version="2.0")
@click.option(
"--config-path", default=DEFAULT_CONFIG_PATH, help="Path to the configuration file"
"--config-path", default=DEFAULT_CONFIG_PATH, help="Path to the configuration file",
)
@click.option("-f", "--folder", help="The folder to download items into.")
@click.option(
@ -50,25 +50,26 @@ def coro(f):
help="Convert the downloaded files to an audio codec (ALAC, FLAC, MP3, AAC, or OGG)",
)
@click.option(
"--no-progress", help="Do not show progress bars", is_flag=True, default=False
"--no-progress", help="Do not show progress bars", is_flag=True, default=False,
)
@click.option(
"-v", "--verbose", help="Enable verbose output (debug mode)", is_flag=True
"-v", "--verbose", help="Enable verbose output (debug mode)", is_flag=True,
)
@click.pass_context
def rip(ctx, config_path, folder, no_db, quality, convert, no_progress, verbose):
"""
Streamrip: the all in one music downloader.
"""Streamrip: the all in one music downloader.
"""
global logger
logging.basicConfig(
level="INFO", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()]
level="INFO", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()],
)
logger = logging.getLogger("streamrip")
if verbose:
install(
console=console,
suppress=[click],
suppress=[
click,
],
show_locals=True,
locals_hide_sunder=False,
)
@ -80,7 +81,7 @@ def rip(ctx, config_path, folder, no_db, quality, convert, no_progress, verbose)
if not os.path.isfile(config_path):
console.print(
f"No file found at [bold cyan]{config_path}[/bold cyan], creating default config."
f"No file found at [bold cyan]{config_path}[/bold cyan], creating default config.",
)
set_user_defaults(config_path)
@ -93,7 +94,7 @@ def rip(ctx, config_path, folder, no_db, quality, convert, no_progress, verbose)
except Exception as e:
console.print(
f"Error loading config from [bold cyan]{config_path}[/bold cyan]: {e}\n"
"Try running [bold]rip config reset[/bold]"
"Try running [bold]rip config reset[/bold]",
)
ctx.obj["config"] = None
return
@ -182,7 +183,7 @@ def config_reset(ctx, yes):
config_path = ctx.obj["config_path"]
if not yes:
if not Confirm.ask(
f"Are you sure you want to reset the config file at {config_path}?"
f"Are you sure you want to reset the config file at {config_path}?",
):
console.print("[green]Reset aborted")
return
@ -242,7 +243,7 @@ def database_browse(ctx, table):
else:
console.print(
f"[red]Invalid database[/red] [bold]{table}[/bold]. [red]Choose[/red] [bold]downloads "
"[red]or[/red] failed[/bold]."
"[red]or[/red] failed[/bold].",
)
@ -259,11 +260,10 @@ def database_browse(ctx, table):
@click.pass_context
@coro
async def search(ctx, first, source, media_type, query):
"""
Search for content using a specific source.
"""Search for content using a specific source.
Example:
-------
rip search qobuz album 'rumours'
"""
with ctx.obj["config"] as cfg:
@ -288,7 +288,6 @@ async def search(ctx, first, source, media_type, query):
@coro
async def lastfm(ctx, source, fallback_source, url):
"""Download tracks from a last.fm playlist using a supported source."""
config = ctx.obj["config"]
if source is not None:
config.session.lastfm.source = source

14
streamrip/rip/main.py
View File

@ -64,7 +64,7 @@ class Main:
client = await self.get_logged_in_client(parsed.source)
self.pending.append(
await parsed.into_pending(client, self.config, self.database)
await parsed.into_pending(client, self.config, self.database),
)
logger.debug("Added url=%s", url)
@ -75,7 +75,7 @@ class Main:
for i, p in enumerate(parsed):
if p is None:
console.print(
f"[red]Found invalid url [cyan]{urls[i]}[/cyan], skipping."
f"[red]Found invalid url [cyan]{urls[i]}[/cyan], skipping.",
)
continue
url_client_pairs.append((p, await self.get_logged_in_client(p.source)))
@ -84,7 +84,7 @@ class Main:
*[
url.into_pending(client, self.config, self.database)
for url, client in url_client_pairs
]
],
)
self.pending.extend(pendings)
@ -93,7 +93,7 @@ class Main:
client = self.clients.get(source)
if client is None:
raise Exception(
f"No client named {source} available. Only have {self.clients.keys()}"
f"No client named {source} available. Only have {self.clients.keys()}",
)
if not client.logged_in:
prompter = get_prompter(client, self.config)
@ -150,7 +150,7 @@ class Main:
assert isinstance(choices, list)
await self.add_all(
[f"http://{source}.com/{media_type}/{item.id}" for item, i in choices]
[f"http://{source}.com/{media_type}/{item.id}" for item, i in choices],
)
else:
@ -177,7 +177,7 @@ class Main:
[
f"http://{source}.com/{item.media_type()}/{item.id}"
for item in choices
]
],
)
async def search_take_first(self, source: str, media_type: str, query: str):
@ -203,7 +203,7 @@ class Main:
fallback_client = None
pending_playlist = PendingLastfmPlaylist(
playlist_url, client, fallback_client, self.config, self.database
playlist_url, client, fallback_client, self.config, self.database,
)
playlist = await pending_playlist.resolve()

13
streamrip/rip/parse_url.py
View File

@ -35,7 +35,7 @@ class URL(ABC):
@abstractmethod
async def into_pending(
self, client: Client, config: Config, db: Database
self, client: Client, config: Config, db: Database,
) -> Pending:
raise NotImplementedError
@ -50,7 +50,7 @@ class GenericURL(URL):
return cls(generic_url, source)
async def into_pending(
self, client: Client, config: Config, db: Database
self, client: Client, config: Config, db: Database,
) -> Pending:
source, media_type, item_id = self.match.groups()
assert client.source == source
@ -80,7 +80,7 @@ class QobuzInterpreterURL(URL):
return cls(qobuz_interpreter_url, "qobuz")
async def into_pending(
self, client: Client, config: Config, db: Database
self, client: Client, config: Config, db: Database,
) -> Pending:
url = self.match.group(0)
artist_id = await self.extract_interpreter_url(url, client)
@ -96,7 +96,7 @@ class QobuzInterpreterURL(URL):
"""
async with client.session.get(url) as resp:
match = QobuzInterpreterURL.interpreter_artist_regex.search(
await resp.text()
await resp.text(),
)
if match:
@ -104,7 +104,7 @@ class QobuzInterpreterURL(URL):
raise Exception(
"Unable to extract artist id from interpreter url. Use a "
"url that contains an artist id."
"url that contains an artist id.",
)
@ -119,7 +119,7 @@ class SoundcloudURL(URL):
self.url = url
async def into_pending(
self, client: SoundcloudClient, config: Config, db: Database
self, client: SoundcloudClient, config: Config, db: Database,
) -> Pending:
resolved = await client._resolve_url(self.url)
media_type = resolved["kind"]
@ -147,6 +147,7 @@ def parse_url(url: str) -> URL | None:
"""Return a URL type given a url string.
Args:
----
url (str): Url to parse
Returns: A URL type, or None if nothing matched.

13
streamrip/rip/prompter.py
View File

@ -23,22 +23,23 @@ class CredentialPrompter(ABC):
@abstractmethod
def has_creds(self) -> bool:
raise NotImplemented
raise NotImplementedError
@abstractmethod
async def prompt_and_login(self):
"""Prompt for credentials in the appropriate way,
and save them to the configuration."""
raise NotImplemented
and save them to the configuration.
"""
raise NotImplementedError
@abstractmethod
def save(self):
"""Save current config to file"""
raise NotImplemented
raise NotImplementedError
@abstractmethod
def type_check_client(self, client: Client):
raise NotImplemented
raise NotImplementedError
class QobuzPrompter(CredentialPrompter):
@ -68,7 +69,7 @@ class QobuzPrompter(CredentialPrompter):
pwd = hashlib.md5(pwd_input.encode("utf-8")).hexdigest()
console.print(
f"[green]Credentials saved to config file at [bold cyan]{self.config.path}"
f"[green]Credentials saved to config file at [bold cyan]{self.config.path}",
)
c = self.config.session.qobuz
c.use_auth_token = False

4
streamrip/rip/validation_regexps.py
View File

@ -1,12 +1,12 @@
import re
URL_REGEX = re.compile(
r"https?://(?:www|open|play|listen)?\.?(qobuz|tidal|deezer)\.com(?:(?:/(album|artist|track|playlist|video|label))|(?:\/[-\w]+?))+\/([-\w]+)"
r"https?://(?:www|open|play|listen)?\.?(qobuz|tidal|deezer)\.com(?:(?:/(album|artist|track|playlist|video|label))|(?:\/[-\w]+?))+\/([-\w]+)",
)
SOUNDCLOUD_URL_REGEX = re.compile(r"https://soundcloud.com/[-\w:/]+")
LASTFM_URL_REGEX = re.compile(r"https://www.last.fm/user/\w+/playlists/\w+")
QOBUZ_INTERPRETER_URL_REGEX = re.compile(
r"https?://www\.qobuz\.com/\w\w-\w\w/interpreter/[-\w]+/[-\w]+"
r"https?://www\.qobuz\.com/\w\w-\w\w/interpreter/[-\w]+/[-\w]+",
)
DEEZER_DYNAMIC_LINK_REGEX = re.compile(r"https://deezer\.page\.link/\w+")
YOUTUBE_URL_REGEX = re.compile(r"https://www\.youtube\.com/watch\?v=[-\w]+")