Fix importing TSV files with S3

vrobbler/apps/scrobbles/models.py

@@ -5,6 +5,7 @@ from typing import Optional
 from uuid import uuid4
 
 from books.models import Book
+from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import models
 from django.urls import reverse
@@ -193,10 +194,16 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
         self.mark_started()
 
         tz = None
+        user_id = None
         if self.user:
+            user_id = self.user.id
             tz = self.user.profile.tzinfo
+        if getattr(settings, "USE_S3_STORAGE"):
+            tsv_str = self.tsv_file.url
+        else:
+            tsv_str = self.tsv_file.path
         scrobbles = process_audioscrobbler_tsv_file(
-            self.tsv_file.path, self.user.id, user_tz=tz
+            tsv_str, user_id, user_tz=tz
         )
         self.record_log(scrobbles)
         self.mark_finished()

vrobbler/apps/scrobbles/tsv.py

@@ -1,8 +1,10 @@
+import codecs
 import csv
 import logging
 from datetime import datetime
 
 import pytz
+import requests
 from music.utils import (
     get_or_create_album,
     get_or_create_artist,
@@ -20,9 +22,16 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
     if not user_tz:
         user_tz = pytz.utc
 
-    with open(file_path) as infile:
+    is_os_file = "https://" not in file_path
+
+    if not is_os_file:
+        r = requests.get(file_path)
+        tsv_data = codecs.iterdecode(r.iter_lines(), "utf-8")
+    else:
+        tsv_data = open(file_path)
+
     source = "Audioscrobbler File"
-        rows = csv.reader(infile, delimiter="\t")
+    rows = csv.reader(tsv_data, delimiter="\t")
 
     source_id = ""
     for row_num, row in enumerate(rows):
@@ -38,9 +47,7 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
             )
             continue
 
-            artist = get_or_create_artist(
+        artist = get_or_create_artist(row[AsTsvColumn["ARTIST_NAME"].value])
-                row[AsTsvColumn["ARTIST_NAME"].value]
-            )
         album = get_or_create_album(
             row[AsTsvColumn["ALBUM_NAME"].value], artist
         )
@@ -50,9 +57,7 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
             mbid=row[AsTsvColumn["MB_ID"].value],
             artist=artist,
             album=album,
-                run_time_seconds=int(
+            run_time_seconds=int(row[AsTsvColumn["RUN_TIME_SECONDS"].value]),
-                    row[AsTsvColumn["RUN_TIME_SECONDS"].value]
-                ),
         )
         if row[AsTsvColumn["COMPLETE"].value] == "S":
             logger.info(
@@ -61,9 +66,7 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
             continue
 
         timestamp = (
-                datetime.utcfromtimestamp(
+            datetime.utcfromtimestamp(int(row[AsTsvColumn["TIMESTAMP"].value]))
-                    int(row[AsTsvColumn["TIMESTAMP"].value])
-                )
             .replace(tzinfo=user_tz)
             .astimezone(pytz.utc)
         )
@@ -86,6 +89,9 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
         logger.debug(f"Queued scrobble {new_scrobble} for creation")
         new_scrobbles.append(new_scrobble)
 
+    if is_os_file:
+        tsv_data.close()
+
     created = Scrobble.objects.bulk_create(new_scrobbles)
     logger.info(
         f"Created {len(created)} scrobbles",

vrobbler/settings.py

@@ -248,7 +248,9 @@ USE_TZ = True
 #
 from storages.backends import s3boto3
 
-if os.getenv("VROBBLER_USE_S3", "False").lower() in TRUTHY:
+USE_S3_STORAGE = os.getenv("VROBBLER_USE_S3", "False").lower() in TRUTHY
+
+if USE_S3_STORAGE:
     AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL", "")
     AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "")
     AWS_S3_ACCESS_KEY_ID = os.getenv("AWS_S3_ACCESS_KEY_ID")