From 4db8793d5cae37e8f9c39c798cb9cdf4054c4aa8 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Sat, 19 Jul 2025 01:54:27 -0400 Subject: [PATCH] [books] Allow timezone changes when importing from KOReader Turns out you need a city-based timezone for DST stuff to work properly. The US/Eastern timezone doesn't mess with DST because it can be so wonky in different regions. So while we fix timezone defaulting to a DST-friendly timezone too. --- vrobbler/apps/books/koreader.py | 91 +++++++++---------- .../0026_userprofile_timezone_change_log.py | 18 ++++ vrobbler/apps/profiles/models.py | 54 ++++++++++- vrobbler/apps/scrobbles/admin.py | 1 + vrobbler/apps/scrobbles/models.py | 13 ++- vrobbler/settings.py | 4 +- 6 files changed, 126 insertions(+), 55 deletions(-) create mode 100644 vrobbler/apps/profiles/migrations/0026_userprofile_timezone_change_log.py diff --git a/vrobbler/apps/books/koreader.py b/vrobbler/apps/books/koreader.py index 6c80b0e..7481576 100644 --- a/vrobbler/apps/books/koreader.py +++ b/vrobbler/apps/books/koreader.py @@ -4,7 +4,8 @@ import sqlite3 from datetime import datetime, timedelta from enum import Enum -import pytz +import pendulum +from zoneinfo import ZoneInfo import requests from books.constants import BOOKS_TITLES_TO_IGNORE from django.apps import apps @@ -208,6 +209,35 @@ def build_page_data(page_rows: list, book_map: dict, user_tz=None) -> dict: ) return book_map +def one_off_fix_colins_profile(profile): + home_tz = "America/New_York" + + europe = "2023-10-15" + europe_tz = "Europe/Paris" + europe_end = "2023-12-15" + + washington = "2023-10-15" + washington_tz = "America/Los_Angeles" + washington_end = "2023-05-04" + + camp = "2024-08-04" + camp_end = "2024-08-10" + camp_tz = "America/Halifax" + + summer = "2025-07-10" + summer_end = "2025-07-13" + summer_tz = "America/Los_Angeles" + + profile.timezone_change_log = "" + profile.timezone_change_log += f"{europe_tz} - {pendulum.parse(europe)}\n" + profile.timezone_change_log += f"{home_tz} - {pendulum.parse(europe_end)}\n" + profile.timezone_change_log += f"{washington_tz} - {pendulum.parse(washington)}\n" + profile.timezone_change_log += f"{home_tz} - {pendulum.parse(washington_end)}\n" + profile.timezone_change_log += f"{camp_tz} - {pendulum.parse(camp)}\n" + profile.timezone_change_log += f"{home_tz} - {pendulum.parse(camp_end)}\n" + profile.timezone_change_log += f"{summer_tz} - {pendulum.parse(summer)}\n" + profile.timezone_change_log += f"{home_tz} - {pendulum.parse(summer_end)}\n" + profile.save() def build_scrobbles_from_book_map( book_map: dict, user: "User" @@ -278,55 +308,18 @@ def build_scrobbles_from_book_map( ) continue - timezone = user.profile.timezone + timestamp = user.profile.get_timestamp_with_tz(datetime.fromtimestamp(int(first_page.get("start_ts")))) + stop_timestamp = user.profile.get_timestamp_with_tz(datetime.fromtimestamp(int(last_page.get("end_ts")))) - timestamp = datetime.fromtimestamp( - int(first_page.get("start_ts")) - ).replace(tzinfo=pytz.timezone(timezone)) + if user.id == 1 and not user.profile.timezone_change_log: + one_off_fix_colins_profile(user.profile) - # Add a shim here temporarily to fix imports while we were in France - # if date is between 10/15 and 12/15, cast it to Europe/Central - if ( - datetime(2023, 10, 15).replace( - tzinfo=pytz.timezone("Europe/Paris") - ) - <= timestamp - <= datetime(2023, 12, 15).replace( - tzinfo=pytz.timezone("Europe/Paris") - ) - ): - timezone = "Europe/Paris" - if ( - datetime(2024, 4, 28).replace( - tzinfo=pytz.timezone("US/Pacific") - ) - <= timestamp - <= datetime(2024, 5, 4).replace( - tzinfo=pytz.timezone("US/Pacific") - ) - ): - timezone = "US/Pacific" - if ( - datetime(2024, 8, 4).replace( - tzinfo=pytz.timezone("Canada/Atlantic") - ) - <= timestamp - <= datetime(2024, 8, 10).replace( - tzinfo=pytz.timezone("Canada/Atlantic") - ) - ): - timezone = "Canada/Atlantic" - - stop_timestamp = datetime.fromtimestamp( - int(last_page.get("end_ts")) - ).replace(tzinfo=pytz.timezone(timezone)) - - if ( - timestamp.tzinfo._dst.seconds == 0 - or stop_timestamp.tzinfo._dst.seconds == 0 - ): + # Adjust for Daylight Saving Time + if timestamp.dst() == timedelta(0) or stop_timestamp.dst() == timedelta(0): timestamp = timestamp - timedelta(hours=1) stop_timestamp = stop_timestamp - timedelta(hours=1) + else: + print("In DST! ", timestamp) scrobble = Scrobble.objects.filter( timestamp=timestamp, @@ -356,7 +349,7 @@ def build_scrobbles_from_book_map( in_progress=False, played_to_completion=True, long_play_complete=False, - timezone=timezone, + timezone=timestamp.tzinfo.name ) ) # Then start over @@ -398,9 +391,9 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: new_scrobbles = [] user = User.objects.filter(id=user_id).first() - tz = pytz.utc + tz = ZoneInfo("UTC") if user: - tz = user.profile.timezone + tz = user.profile.tzinfo is_os_file = "https://" not in file_path if is_os_file: diff --git a/vrobbler/apps/profiles/migrations/0026_userprofile_timezone_change_log.py b/vrobbler/apps/profiles/migrations/0026_userprofile_timezone_change_log.py new file mode 100644 index 0000000..c9fd1b7 --- /dev/null +++ b/vrobbler/apps/profiles/migrations/0026_userprofile_timezone_change_log.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.19 on 2025-07-11 22:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('profiles', '0025_rename_bgstat_id_userprofile_bgstats_id'), + ] + + operations = [ + migrations.AddField( + model_name='userprofile', + name='timezone_change_log', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/vrobbler/apps/profiles/models.py b/vrobbler/apps/profiles/models.py index 9cf3a55..a4874fd 100644 --- a/vrobbler/apps/profiles/models.py +++ b/vrobbler/apps/profiles/models.py @@ -1,4 +1,8 @@ -import pytz +from zoneinfo import ZoneInfo +import pendulum +from datetime import datetime +from django.utils import timezone +import logging from django.conf import settings from django.contrib.auth import get_user_model from django.db import models @@ -10,6 +14,7 @@ from profiles.constants import PRETTY_TIMEZONE_CHOICES User = get_user_model() BNULL = {"blank": True, "null": True} +logger = logging.getLogger(__name__) class UserProfile(TimeStampedModel): user = models.OneToOneField( @@ -18,6 +23,7 @@ class UserProfile(TimeStampedModel): timezone = models.CharField( max_length=255, choices=PRETTY_TIMEZONE_CHOICES, default="UTC" ) + timezone_change_log = models.TextField(**BNULL) lastfm_username = models.CharField(max_length=255, **BNULL) lastfm_password = EncryptedField(**BNULL) lastfm_auto_import = models.BooleanField(default=False) @@ -59,7 +65,51 @@ class UserProfile(TimeStampedModel): @property def tzinfo(self): - return pytz.timezone(self.timezone) + return ZoneInfo(self.timezone) + + def save(self, *args, **kwargs): + if not self._state.adding: + old_instance = UserProfile.objects.get(pk=self.pk) + is_timezone_change = self.timezone != old_instance.timezone + if is_timezone_change: + logger.info("Updating timezone changelog for user", extra={"profile_id": self.id}) + previous_changes = old_instance.timezone_change_log + now = timezone.now().replace(microsecond=0) + new_log = f"{self.timezone} - {now}" + if previous_changes: + new_log = previous_changes + f"\n{new_log}" + self.timezone_change_log = new_log + super(UserProfile, self).save(*args, **kwargs) + + @property + def historic_timezone_changes(self) -> list: + """Return a list of datetimes with timezones for the specific changed time""" + history = [pendulum.datetime(1900, 1, 1, 0, 0, 0, tz=self.tzinfo.key)] + if self.timezone_change_log: + for change in self.timezone_change_log.split("\n"): + if " - " in change: + tz, date = change.split(" - ") + history.append(pendulum.parse(date).in_timezone(tz)) + return history + + def get_timestamp_with_tz(self, timestamp): + timezone = self.tzinfo + if self.timezone_change_log: + change_list = self.historic_timezone_changes + for idx, start in enumerate(change_list): + try: + end = change_list[idx+1] + except IndexError: + end = None + + if end: + if start <= timestamp.replace(tzinfo=end.timezone) <= end: + timezone = start.timezone + else: + if start <= timestamp.replace(tzinfo=start.timezone): + timezone = start.timezone + + return timestamp.replace(tzinfo=timezone) @cached_property def task_context_tags(self) -> list[str]: diff --git a/vrobbler/apps/scrobbles/admin.py b/vrobbler/apps/scrobbles/admin.py index 75a4197..0d77551 100644 --- a/vrobbler/apps/scrobbles/admin.py +++ b/vrobbler/apps/scrobbles/admin.py @@ -139,6 +139,7 @@ class ScrobbleAdmin(admin.ModelAdmin): "media_type", "long_play_complete", "source", + "timezone", ) ordering = ("-timestamp",) diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index ec6f6d8..40e5c1d 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -5,7 +5,7 @@ import logging from collections import defaultdict from typing import Optional from uuid import uuid4 - +from zoneinfo import ZoneInfo import pendulum import pytz from beers.models import Beer @@ -750,7 +750,16 @@ class Scrobble(TimeStampedModel): @property def tzinfo(self): - return pytz.timezone(self.timezone) + return ZoneInfo(self.timezone) + + @property + def local_timestamp(self): + return timezone.localtime(self.timestamp, timezone=self.tzinfo) + + @property + def local_stop_timestamp(self): + if self.stop_tiemstamp: + return timezone.localtime(self.stop_timestamp, timezone=self.tzinfo) @property def scrobble_media_key(self) -> str: diff --git a/vrobbler/settings.py b/vrobbler/settings.py index e1c4546..45d94e2 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -92,7 +92,7 @@ DEFAULT_TASK_CONTEXT_TAGS = [ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -TIME_ZONE = os.getenv("VROBBLER_TIME_ZONE", "US/Eastern") +TIME_ZONE = os.getenv("VROBBLER_TIME_ZONE", "America/New_York") ALLOWED_HOSTS = ["*"] CSRF_TRUSTED_ORIGINS = [ @@ -111,7 +111,7 @@ CELERY_TASK_ALWAYS_EAGER = ( ) CELERY_BROKER_URL = REDIS_URL if REDIS_URL else "memory://localhost/" CELERY_RESULT_BACKEND = "django-db" -CELERY_TIMEZONE = os.getenv("VROBBLER_TIME_ZONE", "US/Eastern") +CELERY_TIMEZONE = os.getenv("VROBBLER_TIME_ZONE", "America/New_York") CELERY_TASK_TRACK_STARTED = True INSTALLED_APPS = [