[importers] Fix scale file checker
This commit is contained in:
@ -507,7 +507,12 @@ def scan_webdav_for_ebird(webdav_client, user_id):
|
||||
|
||||
|
||||
def scan_webdav_for_scale(webdav_client, user_id):
|
||||
"""Download .csv files from WebDAV var/scale/ and queue imports for new files."""
|
||||
"""Download .csv files from WebDAV var/scale/ and queue imports for new files.
|
||||
|
||||
Because the scale CSV always has the same filename but grows by appending
|
||||
rows, we detect changes by downloading the file and comparing its content
|
||||
hash against the last completed import, rather than checking the filename.
|
||||
"""
|
||||
from scrobbles.models import ScaleCSVImport
|
||||
from scrobbles.tasks import process_scale_csv_import
|
||||
|
||||
@ -530,28 +535,41 @@ def scan_webdav_for_scale(webdav_client, user_id):
|
||||
return 0
|
||||
|
||||
new_imports = 0
|
||||
already_imported = set(
|
||||
ScaleCSVImport.objects.filter(user_id=user_id).values_list(
|
||||
"original_filename", flat=True
|
||||
)
|
||||
)
|
||||
|
||||
for fname in files:
|
||||
fname = os.path.basename(fname)
|
||||
if not fname.lower().endswith(".csv"):
|
||||
continue
|
||||
if fname in already_imported:
|
||||
logger.debug(f"Skipping already-imported {fname}")
|
||||
continue
|
||||
|
||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
|
||||
try:
|
||||
webdav_client.download_sync(
|
||||
remote_path=f"{scale_path}/{fname}", local_path=tmp.name
|
||||
)
|
||||
|
||||
new_hash = get_file_md5_hash(tmp.name)
|
||||
|
||||
last_import = (
|
||||
ScaleCSVImport.objects.filter(
|
||||
user_id=user_id,
|
||||
original_filename=fname,
|
||||
processed_finished__isnull=False,
|
||||
)
|
||||
.order_by("-processed_finished")
|
||||
.first()
|
||||
)
|
||||
if last_import and last_import.file_hash == new_hash:
|
||||
logger.debug(
|
||||
"Scale CSV %s unchanged (hash match), skipping",
|
||||
fname,
|
||||
extra={"user_id": user_id, "hash": new_hash},
|
||||
)
|
||||
continue
|
||||
|
||||
imp = ScaleCSVImport.objects.create(
|
||||
user_id=user_id,
|
||||
original_filename=fname,
|
||||
file_hash=new_hash,
|
||||
)
|
||||
with open(tmp.name, "rb") as f:
|
||||
imp.csv_file.save(fname, f, save=True)
|
||||
|
||||
@ -0,0 +1,18 @@
|
||||
# Generated by Django 4.2.29 on 2026-05-28
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
("scrobbles", "0087_koreaderimport_webdav_etag"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="scalecsvimport",
|
||||
name="file_hash",
|
||||
field=models.CharField(blank=True, max_length=32, null=True),
|
||||
),
|
||||
]
|
||||
@ -287,6 +287,7 @@ class ScaleCSVImport(BaseFileImportMixin):
|
||||
|
||||
csv_file = models.FileField(upload_to=get_path, **BNULL)
|
||||
original_filename = models.CharField(max_length=255, **BNULL)
|
||||
file_hash = models.CharField(max_length=32, **BNULL)
|
||||
|
||||
def process(self, force=False):
|
||||
from scrobbles.importers.scale import import_scale_csv
|
||||
|
||||
Reference in New Issue
Block a user