[books] Fix webdav importer

PROJECT.org

@@ -79,20 +79,7 @@ fetching and simple saving.
 :LOGBOOK:
 CLOCK: [2025-07-09 Wed 09:55]--[2025-07-09 Wed 10:15] =>  0:20
 :END:
-* Backlog [4/27]
-** STRT Only create a LastFM import if there are files to import :vrobbler:feature:lastfm:importers:project:personal:
-:PROPERTIES:
-:ID:       7a1456af-c5f1-6385-b34f-0be24a6b65b0
-:END:
-- Note taken on [2025-07-20 Sun 16:21]
-
-  This thing is kicking my butt. As it stands it works, but the scrobbles are not assigned to the tracks properly.
-** TODO Add timezone awarness to IMAP importer :personal:project:vrobbler:feature:importer:imap:timezones:
-** DONE Track timezone changes for profiles :vrobbler:feature:profiles:personal:project:
-:PROPERTIES:
-:ID:       89ec867f-29fd-82f1-be17-b49dddc30c78
-:END:
-[2025-07-11 14:23]
+* Backlog [7/28]
 ** TODO [#A] Add classmethod for metadata fetching to tracks :vrobbler:feature:music:personal:project:
 :PROPERTIES:
 :ID:       bc4b45e5-4c65-13c5-ab7b-1937d3fbf5c2
@@ -456,6 +443,29 @@ it's annoying.
 ** TODO [#C] Allow users to see tasks on calendar view :vrobbler:personal:project:templates:feature:
 https://codepen.io/oliviale/pen/QYqybo
 ** TODO [#C] Come up with a possible flow using WebDAV and super-productivity for tasks :personal:feature:project:vrobbler:tasks:
+* Version 18.4
+** DONE Track timezone changes for profiles :vrobbler:feature:profiles:personal:project:
+:PROPERTIES:
+:ID:       89ec867f-29fd-82f1-be17-b49dddc30c78
+:END:
+[2025-07-11 14:23]
+** DONE Only create a LastFM import if there are files to import :vrobbler:feature:lastfm:importers:project:personal:
+:PROPERTIES:
+:ID:       7a1456af-c5f1-6385-b34f-0be24a6b65b0
+:END:
+- Note taken on [2025-07-20 Sun 16:21]
+
+  This thing is kicking my butt. As it stands it works, but the scrobbles are not assigned to the tracks properly.
+* Version 18.3
+** DONE Add timezone awarness to IMAP importer :personal:project:vrobbler:feature:importer:imap:timezones:
+:PROPERTIES:
+:ID:       05837b7c-96aa-6190-3678-e2ae7c7cac75
+:END:
+* Version 18
+** DONE Condense tracks of the same title by the same artist with multiple albums :vrobbler:feature:music:project:personal:
+:PROPERTIES:
+:ID:       b39fcec8-59fd-eab0-5809-b8144c7d2708
+:END:
 ** DONE Import from BG stats a "learning" log field when "Learning to play" is in the comment :vrobbler:feature:boardgames:project:personal:
 :PROPERTIES:
 :ID:       fda59fab-4349-e99e-54c6-9f1392a1c474

vrobbler/apps/books/koreader.py

@@ -3,16 +3,15 @@ import re
 import sqlite3
 from datetime import datetime, timedelta
 from enum import Enum
-
-import pendulum
 from zoneinfo import ZoneInfo
+
 import requests
 from books.constants import BOOKS_TITLES_TO_IGNORE
 from django.apps import apps
 from django.contrib.auth import get_user_model
-from stream_sqlite import stream_sqlite
+from profiles.utils import one_off_fix_colins_profile
 from scrobbles.notifications import NtfyNotification
-from vrobbler.apps.profiles.utils import one_off_fix_colins_profile
+from stream_sqlite import stream_sqlite
 from webdav.client import get_webdav_client
 
 logger = logging.getLogger(__name__)

vrobbler/apps/profiles/admin.py

@@ -7,6 +7,7 @@ from profiles.models import UserProfile
 class UserProfileAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
     ordering = ("-created",)
+    readonly_fields = ("timezone_change_log",)
     exclude = (
         "twitch_token",
         "twitch_client_secret",

vrobbler/apps/profiles/models.py

@@ -16,6 +16,7 @@ BNULL = {"blank": True, "null": True}
 
 logger = logging.getLogger(__name__)
 
+
 class UserProfile(TimeStampedModel):
     user = models.OneToOneField(
         User, on_delete=models.CASCADE, related_name="profile"
@@ -72,7 +73,10 @@ class UserProfile(TimeStampedModel):
             old_instance = UserProfile.objects.get(pk=self.pk)
             is_timezone_change = self.timezone != old_instance.timezone
             if is_timezone_change:
-                logger.info("Updating timezone changelog for user", extra={"profile_id": self.id})
+                logger.info(
+                    "Updating timezone changelog for user",
+                    extra={"profile_id": self.id},
+                )
                 previous_changes = old_instance.timezone_change_log
                 now = timezone.now().replace(microsecond=0)
                 new_log = f"{self.timezone} - {now}"
@@ -98,7 +102,7 @@ class UserProfile(TimeStampedModel):
             change_list = self.historic_timezone_changes
             for idx, start in enumerate(change_list):
                 try:
-                    end = change_list[idx+1]
+                    end = change_list[idx + 1]
                 except IndexError:
                     end = None
 

vrobbler/apps/profiles/utils.py

@@ -62,20 +62,20 @@ def start_of_year(dt, profile) -> datetime:
 def one_off_fix_colins_profile(profile):
     home_tz = "America/New_York"
 
-    europe = "2022-10-15"
+    europe = "2022-10-15 06:00:00"
+    europe_end = "2023-12-16 12:00:00"
     europe_tz = "Europe/Paris"
-    europe_end = "2023-12-15"
 
-    washington = "2023-04-28"
+    washington = "2023-04-28 06:00:00"
+    washington_end = "2023-05-04 12:00:00"
     washington_tz = "America/Los_Angeles"
-    washington_end = "2023-05-04"
 
-    camp = "2024-08-04"
-    camp_end = "2024-08-10"
+    camp = "2024-08-04 17:00:00"
+    camp_end = "2024-08-10 12:00:00"
     camp_tz = "America/Halifax"
 
-    summer = "2025-07-09 12:00:00"
-    summer_end = "2025-07-13 20:30:00"
+    summer = "2025-07-09 06:00:00"
+    summer_end = "2025-07-11 23:30:00"
     summer_tz = "America/Los_Angeles"
 
     profile.timezone_change_log = ""

vrobbler/apps/scrobbles/importers/lastfm.py

@@ -50,20 +50,22 @@ class LastFM:
                 enrich=True,
             )
 
-            timezone = settings.TIME_ZONE
-            if self.vrobbler_user.profile:
-                timezone = self.vrobbler_user.profile.timezone
-
-            timestamp = lfm_scrobble.get("timestamp")
+            timestamp = self.vrobbler_user.profile.get_timestamp_with_tz(
+                lfm_scrobble.get("timestamp")
+            )
+            stop_timestamp = timestamp + timedelta(
+                seconds=track.run_time_seconds
+            )
             new_scrobble = Scrobble(
                 user=self.vrobbler_user,
                 timestamp=timestamp,
+                stop_timestamp=stop_timestamp,
                 source=source,
                 track=track,
-                timezone=timezone,
                 played_to_completion=True,
                 in_progress=False,
                 media_type=Scrobble.MediaType.TRACK,
+                timezone=timestamp.tzinfo.name,
             )
             # Vrobbler scrobbles on finish, LastFM scrobbles on start
             seconds_eariler = timestamp - timedelta(seconds=20)
@@ -91,7 +93,7 @@ class LastFM:
         )
         return created
 
-    def get_last_scrobbles(self, time_from=None, time_to=None):
+    def get_last_scrobbles(self, time_from=None, time_to=None, check=False):
         """Given a user, Last.fm api key, and secret key, grab a list of scrobbled
         tracks"""
         lfm_params = {}
@@ -107,6 +109,9 @@ class LastFM:
         found_scrobbles = self.user.get_recent_tracks(**lfm_params)
         # TOOD spin this out into a celery task over certain threshold of found scrobbles?
 
+        if check and found_scrobbles:
+            return True
+
         for scrobble in found_scrobbles:
             logger.info(f"Processing {scrobble}")
             run_time = None

vrobbler/apps/scrobbles/importers/webdav.py

@@ -42,7 +42,7 @@ def import_from_webdav_for_all_users(restart=False):
                 KoReaderImport.objects.filter(
                     user_id=user_id, processed_finished__isnull=False
                 )
-                .order_by("Processed_finished")
+                .order_by("processed_finished")
                 .last()
             )
 

vrobbler/apps/scrobbles/models.py

@@ -1087,6 +1087,7 @@ class Scrobble(TimeStampedModel):
         key = media_class_to_foreign_key(media.__class__.__name__)
         media_query = models.Q(**{key: media})
         scrobble_data[key + "_id"] = media.id
+        skip_in_progress_check = kwargs.get("skip_in_progress_check", False)
 
         # Find our last scrobble of this media item (track, video, etc)
         scrobble = (
@@ -1110,27 +1111,31 @@ class Scrobble(TimeStampedModel):
             )
             return scrobble
 
-        logger.info(
-            f"[create_or_update] check for existing scrobble to update ",
-            extra={
-                "scrobble_id": scrobble.id if scrobble else None,
-                "media_type": mtype,
-                "media_id": media.id,
-                "scrobble_data": scrobble_data,
-            },
-        )
-        scrobble_data["playback_status"] = scrobble_data.pop("status", None)
-        # If it's marked as stopped, send it through our update mechanism, which will complete it
-        if scrobble and (
-            scrobble.can_be_updated
-            or scrobble_data["playback_status"] == "stopped"
-        ):
-            if "log" in scrobble_data.keys() and scrobble.log:
-                scrobble_data["log"] = scrobble.log | scrobble_data["log"]
-            return scrobble.update(scrobble_data)
+        if not skip_in_progress_check:
+            logger.info(
+                f"[create_or_update] check for existing scrobble to update ",
+                extra={
+                    "scrobble_id": scrobble.id if scrobble else None,
+                    "media_type": mtype,
+                    "media_id": media.id,
+                    "scrobble_data": scrobble_data,
+                },
+            )
+            scrobble_data["playback_status"] = scrobble_data.pop(
+                "status", None
+            )
+            # If it's marked as stopped, send it through our update mechanism, which will complete it
+            if scrobble and (
+                scrobble.can_be_updated
+                or scrobble_data["playback_status"] == "stopped"
+            ):
+                if "log" in scrobble_data.keys() and scrobble.log:
+                    scrobble_data["log"] = scrobble.log | scrobble_data["log"]
+                return scrobble.update(scrobble_data)
+
+            # Discard status before creating
+            scrobble_data.pop("playback_status")
 
-        # Discard status before creating
-        scrobble_data.pop("playback_status")
         logger.info(
             f"[scrobbling] creating new scrobble",
             extra={

vrobbler/apps/scrobbles/scrobblers.py

@@ -18,6 +18,7 @@ from music.models import Track
 from people.models import Person
 from podcasts.models import PodcastEpisode
 from podcasts.utils import parse_mopidy_uri
+from profiles.models import UserProfile
 from puzzles.models import Puzzle
 from scrobbles.constants import (
     JELLYFIN_AUDIO_ITEM_TYPES,
@@ -25,6 +26,7 @@ from scrobbles.constants import (
     SCROBBLE_CONTENT_URLS,
 )
 from scrobbles.models import Scrobble
+from scrobbles.notifications import NtfyNotification
 from scrobbles.utils import convert_to_seconds, extract_domain
 from sports.models import SportEvent
 from sports.thesportsdb import lookup_event_from_thesportsdb
@@ -390,10 +392,25 @@ def email_scrobble_board_game(
         locations[location_dict.get("id")] = location
 
     scrobbles_created = []
+    second = 0
     for play_dict in bgstat_data.get("plays", []):
+        hour = None
+        minute = None
+        second = None
         if "comments" in play_dict.keys():
-            if "Learning to play" in play_dict.get("comments"):
-                log_data["learning"] = True
+            for line in play_dict.get("comments", "").split("\n"):
+                if "Learning to play" in line:
+                    log_data["learning"] = True
+                if "Start time:" in line:
+                    start_time = line.split(": ")[1]
+                    pieces = start_time.split(":")
+                    hour = int(pieces[0])
+                    minute = int(pieces[1])
+                    try:
+                        second = int(pieces[2])
+                    except IndexError:
+                        second = 0
+
             log_data["details"] = play_dict.get("comments")
         log_data["expansion_ids"] = []
         try:
@@ -402,7 +419,6 @@ def email_scrobble_board_game(
             try:
                 base_game = expansions[play_dict.get("gameRefId")]
             except KeyError:
-                print(play_dict)
                 logger.info(
                     "Skipping scrobble of play, can't find game",
                     extra={"play_dict": play_dict},
@@ -440,35 +456,56 @@ def email_scrobble_board_game(
                 }
             )
 
-        start = parse(play_dict.get("playDate"))
+        timestamp = parse(play_dict.get("playDate"))
+        if hour and minute:
+            logger.info(f"Scrobble playDate has manual start time {timestamp}")
+            timestamp = timestamp.replace(
+                hour=hour, minute=minute, second=second or 0
+            )
+            logger.info(f"Update to {timestamp}")
+
+        profile = UserProfile.objects.filter(user_id=user_id).first()
+        timestamp = profile.get_timestamp_with_tz(timestamp)
+
         if play_dict.get("durationMin") > 0:
             duration_seconds = play_dict.get("durationMin") * 60
         else:
             duration_seconds = base_game.run_time_seconds
-        stop = start + timedelta(seconds=duration_seconds)
+        stop_timestamp = timestamp + timedelta(seconds=duration_seconds)
+
+        logger.info(f"Creating scrobble for {base_game} at {timestamp}")
         scrobble_dict = {
             "user_id": user_id,
-            "timestamp": start,
+            "timestamp": timestamp,
             "playback_position_seconds": duration_seconds,
             "source": "BG Stats",
             "log": log_data,
         }
 
-        scrobble = Scrobble.objects.filter(
-            board_game=base_game, user_id=user_id, timestamp=start
-        ).first()
+        scrobble = None
+        if timestamp.year > 2023:
+            logger.info(
+                "Scrobbles older than 2024 likely have no time associated just create it"
+            )
+            scrobble = Scrobble.objects.filter(
+                board_game=base_game, user_id=user_id, timestamp=timestamp
+            ).first()
         if scrobble:
             logger.info(
                 "Scrobble already exists, skipping",
                 extra={"scrobble_dict": scrobble_dict, "user_id": user_id},
             )
             continue
-        scrobble = Scrobble.create_or_update(base_game, user_id, scrobble_dict)
-        scrobble.stop_timestamp = stop
+        scrobble = Scrobble.create_or_update(
+            base_game, user_id, scrobble_dict, skip_in_progress_check=True
+        )
+        scrobble.timezone = timestamp.tzinfo.name
+        scrobble.stop_timestamp = stop_timestamp
         scrobble.in_progress = False
         scrobble.played_to_completion = True
         scrobble.save()
         scrobbles_created.append(scrobble)
+        NtfyNotification(scrobble).send()
 
     return scrobbles_created
 

vrobbler/apps/scrobbles/utils.py

@@ -114,6 +114,8 @@ def get_long_plays_completed(user: User) -> list:
 
 def import_lastfm_for_all_users(restart=False):
     """Grab a list of all users with LastFM enabled and kickoff imports for them"""
+    from scrobbles.importers.lastfm import LastFM
+
     LastFmImport = apps.get_model("scrobbles", "LastFMImport")
     lastfm_enabled_user_ids = UserProfile.objects.filter(
         lastfm_username__isnull=False,
@@ -124,6 +126,31 @@ def import_lastfm_for_all_users(restart=False):
     lastfm_import_count = 0
 
     for user_id in lastfm_enabled_user_ids:
+
+        lfm_import = LastFmImport.objects.filter(
+            user_id=user_id, processed_finished__isnull=False
+        ).last()
+        if lfm_import:
+            last_processed = lfm_import.processed_finished
+        else:
+            logger.info(
+                f"Not resuming failed LastFM import {lfm_import.id} for user {user_id}, use restart=True to restart"
+                "No existing LastFM import, we should start a monthly parsing of lastFm for this user going back to 2002"
+            )
+            continue
+
+        lfm_client = LastFM(
+            user=get_user_model().objects.filter(id=user_id).first()
+        )
+
+        has_scrobbles = lfm_client.get_last_scrobbles(
+            time_from=last_processed, check=True
+        )
+
+        if not has_scrobbles:
+            logger.info("No new scrobbles to import from LastFM")
+            continue
+
         lfm_import, created = LastFmImport.objects.get_or_create(
             user_id=user_id, processed_finished__isnull=True
         )