[music] Fix historical LFM imports

vrobbler/apps/scrobbles/importers/lastfm.py

@@ -1,9 +1,14 @@
+import calendar
 import logging
 from datetime import datetime, timedelta
 
 import pylast
 import pytz
+from celery import shared_task
+from dateutil.relativedelta import relativedelta
 from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.utils import timezone
 from music.models import Track
 
 logger = logging.getLogger(__name__)
@@ -22,6 +27,8 @@ PYLAST_ERRORS = tuple(
 
 class LastFM:
     def __init__(self, user):
+        self.user = None
+        self.vrobbler_user = user
         try:
             self.client = pylast.LastFMNetwork(
                 api_key=getattr(settings, "LASTFM_API_KEY"),
@@ -30,24 +37,28 @@ class LastFM:
                 password_hash=pylast.md5(user.profile.lastfm_password),
             )
             self.user = self.client.get_user(user.profile.lastfm_username)
-            self.vrobbler_user = user
         except PYLAST_ERRORS as e:
             logger.error(f"Error during Last.fm setup: {e}")
+            raise
 
-    def import_from_lastfm(self, last_processed=None):
+    def import_from_lastfm(self, last_processed=None, time_to=None):
         """Given a last processed time, import all scrobbles from LastFM since then"""
         from scrobbles.models import Scrobble
 
         new_scrobbles = []
         source = "Last.fm"
-        lastfm_scrobbles = self.get_last_scrobbles(time_from=last_processed)
+        lastfm_scrobbles = self.get_last_scrobbles(
+            time_from=last_processed, time_to=time_to
+        )
 
         for lfm_scrobble in lastfm_scrobbles:
             track = Track.find_or_create(
                 title=lfm_scrobble.get("title"),
                 artist_name=lfm_scrobble.get("artist"),
-                album_name=lfm_scrobble.get("album"),
-                enrich=True,
+                album_name=lfm_scrobble.get("album", ""),
+                run_time_seconds=lfm_scrobble.get("run_time_seconds"),
+                mbid=lfm_scrobble.get("mbid"),
+                enrich=False,
             )
 
             tz_timestamp = self.vrobbler_user.profile.get_timestamp_with_tz(
@@ -92,6 +103,20 @@ class LastFM:
         )
         return created
 
+    def get_earliest_scrobble_timestamp(self) -> datetime | None:
+        """Return the user's Last.fm registration date as the earliest
+        possible scrobble timestamp.
+
+        Uses ``get_unixtime_registered()`` which is a single API call.
+        Returns ``None`` if the user has no scrobbles at all.
+        """
+        if not self.get_last_scrobbles(check=True):
+            return None
+        registered_ts = self.user.get_unixtime_registered()
+        earliest = datetime.fromtimestamp(registered_ts, tz=pytz.utc)
+        logger.info("Last.fm user %s registered on %s", self.user.name, earliest)
+        return earliest
+
     def get_last_scrobbles(self, time_from=None, time_to=None, check=False):
         """Given a user, Last.fm api key, and secret key, grab a list of scrobbled
         tracks"""
@@ -102,14 +127,12 @@ class LastFM:
         if time_to:
             lfm_params["time_to"] = int(time_to.timestamp())
 
-        # if not time_from and not time_to:
-        lfm_params["limit"] = None
+        lfm_params["limit"] = 1 if check else None
 
         found_scrobbles = self.user.get_recent_tracks(**lfm_params)
-        # TOOD spin this out into a celery task over certain threshold of found scrobbles?
 
-        if check and found_scrobbles:
-            return True
+        if check:
+            return bool(found_scrobbles)
 
         for scrobble in found_scrobbles:
             logger.info(f"Processing {scrobble}")
@@ -163,3 +186,112 @@ class LastFM:
                 }
             )
         return scrobbles
+
+
+@shared_task
+def dispatch_historical_imports(user_id):
+    """Find every un-imported month and dispatch a sub-task for each.
+
+    Walks backward month by month from the user's earliest completed
+    ``LastFmImport`` (or the current month if none exists) to their
+    first Last.fm scrobble, creating a ``LastFmImport`` record and
+    dispatching ``process_lastfm_import`` for each month that has
+    scrobbles.
+    """
+    from scrobbles.models import LastFmImport
+    from scrobbles.tasks import process_lastfm_import
+
+    user = get_user_model().objects.filter(id=user_id).first()
+    if not user:
+        logger.error("User %s not found", user_id)
+        return 0
+
+    try:
+        lastfm = LastFM(user)
+    except Exception:
+        logger.exception("Failed to set up LastFM client for user %s", user_id)
+        return 0
+
+    logger.info("Looking up earliest Last.fm scrobble for user %s", user_id)
+    earliest_scrobble = lastfm.get_earliest_scrobble_timestamp()
+    if earliest_scrobble is None:
+        logger.info("User %s has no Last.fm scrobbles", user_id)
+        return 0
+    logger.info("Earliest Last.fm scrobble for user %s: %s", user_id, earliest_scrobble)
+
+    in_progress = LastFmImport.objects.filter(
+        user_id=user_id, processed_finished__isnull=True
+    ).exists()
+    if in_progress:
+        logger.info(
+            "User %s has an import still in progress, skipping dispatch",
+            user_id,
+        )
+        return 0
+
+    def _first_of_month(dt):
+        return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+
+    def _last_of_month(dt):
+        _, last_day = calendar.monthrange(dt.year, dt.month)
+        return dt.replace(
+            day=last_day, hour=23, minute=59, second=59, microsecond=999999
+        )
+
+    earliest = (
+        LastFmImport.objects.filter(user_id=user_id, processed_finished__isnull=False)
+        .order_by("processed_finished")
+        .first()
+    )
+    if earliest:
+        earliest_log_scrobble = earliest.scrobbles().order_by("timestamp").first()
+        cursor = (
+            _first_of_month(earliest_log_scrobble.timestamp)
+            if earliest_log_scrobble
+            else earliest.processed_finished
+        )
+        logger.info(
+            "Found existing import; earliest scrobble %s, cursor set to %s",
+            earliest_log_scrobble.timestamp if earliest_log_scrobble else None,
+            cursor,
+        )
+    else:
+        cursor = timezone.now()
+        logger.info("No existing imports; cursor set to %s", cursor)
+
+    dispatched = 0
+    while True:
+        cursor = _first_of_month(cursor) - relativedelta(days=1)
+        month_start = _first_of_month(cursor)
+        month_end = _last_of_month(cursor)
+
+        if month_end < earliest_scrobble:
+            logger.info("Reached earliest scrobble at %s, stopping", earliest_scrobble)
+            break
+
+        logger.info(
+            "Checking for Last.fm scrobbles in range %s to %s",
+            month_start.date(),
+            month_end.date(),
+        )
+
+        has = lastfm.get_last_scrobbles(
+            time_from=month_start, time_to=month_end, check=True
+        )
+        if not has:
+            continue
+
+        lfm_import = LastFmImport.objects.create(user_id=user_id)
+        process_lastfm_import.delay(
+            lfm_import.id, time_from=month_start, time_to=month_end
+        )
+        logger.info(
+            "%s – %s dispatched (import %s)",
+            month_start.date(),
+            month_end.date(),
+            lfm_import.id,
+        )
+        dispatched += 1
+
+    logger.info("Dispatched %d monthly batch(es) for user %s", dispatched, user_id)
+    return dispatched