[importers] Fix scale file checker
This commit is contained in:
@ -507,7 +507,12 @@ def scan_webdav_for_ebird(webdav_client, user_id):
|
|||||||
|
|
||||||
|
|
||||||
def scan_webdav_for_scale(webdav_client, user_id):
|
def scan_webdav_for_scale(webdav_client, user_id):
|
||||||
"""Download .csv files from WebDAV var/scale/ and queue imports for new files."""
|
"""Download .csv files from WebDAV var/scale/ and queue imports for new files.
|
||||||
|
|
||||||
|
Because the scale CSV always has the same filename but grows by appending
|
||||||
|
rows, we detect changes by downloading the file and comparing its content
|
||||||
|
hash against the last completed import, rather than checking the filename.
|
||||||
|
"""
|
||||||
from scrobbles.models import ScaleCSVImport
|
from scrobbles.models import ScaleCSVImport
|
||||||
from scrobbles.tasks import process_scale_csv_import
|
from scrobbles.tasks import process_scale_csv_import
|
||||||
|
|
||||||
@ -530,28 +535,41 @@ def scan_webdav_for_scale(webdav_client, user_id):
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
new_imports = 0
|
new_imports = 0
|
||||||
already_imported = set(
|
|
||||||
ScaleCSVImport.objects.filter(user_id=user_id).values_list(
|
|
||||||
"original_filename", flat=True
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
for fname in files:
|
for fname in files:
|
||||||
fname = os.path.basename(fname)
|
fname = os.path.basename(fname)
|
||||||
if not fname.lower().endswith(".csv"):
|
if not fname.lower().endswith(".csv"):
|
||||||
continue
|
continue
|
||||||
if fname in already_imported:
|
|
||||||
logger.debug(f"Skipping already-imported {fname}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
|
||||||
try:
|
try:
|
||||||
webdav_client.download_sync(
|
webdav_client.download_sync(
|
||||||
remote_path=f"{scale_path}/{fname}", local_path=tmp.name
|
remote_path=f"{scale_path}/{fname}", local_path=tmp.name
|
||||||
)
|
)
|
||||||
|
|
||||||
|
new_hash = get_file_md5_hash(tmp.name)
|
||||||
|
|
||||||
|
last_import = (
|
||||||
|
ScaleCSVImport.objects.filter(
|
||||||
|
user_id=user_id,
|
||||||
|
original_filename=fname,
|
||||||
|
processed_finished__isnull=False,
|
||||||
|
)
|
||||||
|
.order_by("-processed_finished")
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if last_import and last_import.file_hash == new_hash:
|
||||||
|
logger.debug(
|
||||||
|
"Scale CSV %s unchanged (hash match), skipping",
|
||||||
|
fname,
|
||||||
|
extra={"user_id": user_id, "hash": new_hash},
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
imp = ScaleCSVImport.objects.create(
|
imp = ScaleCSVImport.objects.create(
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
original_filename=fname,
|
original_filename=fname,
|
||||||
|
file_hash=new_hash,
|
||||||
)
|
)
|
||||||
with open(tmp.name, "rb") as f:
|
with open(tmp.name, "rb") as f:
|
||||||
imp.csv_file.save(fname, f, save=True)
|
imp.csv_file.save(fname, f, save=True)
|
||||||
|
|||||||
@ -0,0 +1,18 @@
|
|||||||
|
# Generated by Django 4.2.29 on 2026-05-28
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("scrobbles", "0087_koreaderimport_webdav_etag"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="scalecsvimport",
|
||||||
|
name="file_hash",
|
||||||
|
field=models.CharField(blank=True, max_length=32, null=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
@ -287,6 +287,7 @@ class ScaleCSVImport(BaseFileImportMixin):
|
|||||||
|
|
||||||
csv_file = models.FileField(upload_to=get_path, **BNULL)
|
csv_file = models.FileField(upload_to=get_path, **BNULL)
|
||||||
original_filename = models.CharField(max_length=255, **BNULL)
|
original_filename = models.CharField(max_length=255, **BNULL)
|
||||||
|
file_hash = models.CharField(max_length=32, **BNULL)
|
||||||
|
|
||||||
def process(self, force=False):
|
def process(self, force=False):
|
||||||
from scrobbles.importers.scale import import_scale_csv
|
from scrobbles.importers.scale import import_scale_csv
|
||||||
|
|||||||
Reference in New Issue
Block a user