[importers] Fix scale file checker

This commit is contained in:
2026-05-28 17:05:48 -04:00
parent 034cb99c77
commit bea2b2d187
3 changed files with 46 additions and 9 deletions

View File

@ -507,7 +507,12 @@ def scan_webdav_for_ebird(webdav_client, user_id):
def scan_webdav_for_scale(webdav_client, user_id):
"""Download .csv files from WebDAV var/scale/ and queue imports for new files."""
"""Download .csv files from WebDAV var/scale/ and queue imports for new files.
Because the scale CSV always has the same filename but grows by appending
rows, we detect changes by downloading the file and comparing its content
hash against the last completed import, rather than checking the filename.
"""
from scrobbles.models import ScaleCSVImport
from scrobbles.tasks import process_scale_csv_import
@ -530,28 +535,41 @@ def scan_webdav_for_scale(webdav_client, user_id):
return 0
new_imports = 0
already_imported = set(
ScaleCSVImport.objects.filter(user_id=user_id).values_list(
"original_filename", flat=True
)
)
for fname in files:
fname = os.path.basename(fname)
if not fname.lower().endswith(".csv"):
continue
if fname in already_imported:
logger.debug(f"Skipping already-imported {fname}")
continue
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
try:
webdav_client.download_sync(
remote_path=f"{scale_path}/{fname}", local_path=tmp.name
)
new_hash = get_file_md5_hash(tmp.name)
last_import = (
ScaleCSVImport.objects.filter(
user_id=user_id,
original_filename=fname,
processed_finished__isnull=False,
)
.order_by("-processed_finished")
.first()
)
if last_import and last_import.file_hash == new_hash:
logger.debug(
"Scale CSV %s unchanged (hash match), skipping",
fname,
extra={"user_id": user_id, "hash": new_hash},
)
continue
imp = ScaleCSVImport.objects.create(
user_id=user_id,
original_filename=fname,
file_hash=new_hash,
)
with open(tmp.name, "rb") as f:
imp.csv_file.save(fname, f, save=True)

View File

@ -0,0 +1,18 @@
# Generated by Django 4.2.29 on 2026-05-28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("scrobbles", "0087_koreaderimport_webdav_etag"),
]
operations = [
migrations.AddField(
model_name="scalecsvimport",
name="file_hash",
field=models.CharField(blank=True, max_length=32, null=True),
),
]

View File

@ -287,6 +287,7 @@ class ScaleCSVImport(BaseFileImportMixin):
csv_file = models.FileField(upload_to=get_path, **BNULL)
original_filename = models.CharField(max_length=255, **BNULL)
file_hash = models.CharField(max_length=32, **BNULL)
def process(self, force=False):
from scrobbles.importers.scale import import_scale_csv