150 lines
5.0 KiB
Python
150 lines
5.0 KiB
Python
import json
|
|
import logging
|
|
import re
|
|
|
|
import requests
|
|
import yake
|
|
|
|
CROSSREF_WORK_URL = "https://api.crossref.org/works/{}"
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_STOPWORDS = {
|
|
"this", "that", "these", "those", "the", "a", "an", "in", "on", "at",
|
|
"to", "for", "of", "and", "or", "is", "are", "was", "were", "be",
|
|
"been", "being", "have", "has", "had", "do", "does", "did", "will",
|
|
"would", "can", "could", "may", "might", "shall", "should", "not",
|
|
"no", "nor", "with", "from", "by", "as", "at", "but", "if", "because",
|
|
"while", "although", "however", "we", "our", "their", "its", "it",
|
|
"they", "them", "also", "more", "most", "new", "such", "into",
|
|
"across", "between", "through", "about", "after", "before", "during",
|
|
"within", "without", "other", "many", "some", "each", "every", "both",
|
|
"few", "own", "via",
|
|
}
|
|
|
|
_DROP_PHRASES = {
|
|
"paper", "study", "studies", "research", "introduction", "conclusion",
|
|
"conclusions", "background", "methods", "results", "findings",
|
|
"analysis", "approach", "approaches", "framework", "theory",
|
|
"theories", "concept", "concepts", "model", "models", "process",
|
|
"processes", "role", "roles", "factor", "factors", "effect",
|
|
"effects", "impact", "implication", "implications", "actor", "actors",
|
|
"article", "chapter", "section", "discussion", "review", "overview",
|
|
"summary", "methodology", "special issue", "implications",
|
|
"limitations", "findings", "purpose", "objective", "objectives",
|
|
"design", "setting", "participants", "sample", "data",
|
|
"contemporary", "little", "empirical", "theoretical",
|
|
"organizations", "dissent",
|
|
}
|
|
|
|
|
|
def _strip_jats(text: str) -> str:
|
|
if not text:
|
|
return ""
|
|
text = re.sub(r"</?jats:[^>]*>", "", text)
|
|
text = re.sub(r"^\s*Abstract\s*", "", text)
|
|
return text.strip()
|
|
|
|
|
|
def _extract_genres_from_abstract(abstract: str, max_keywords: int = 8) -> list[str]:
|
|
if not abstract or len(abstract) < 50:
|
|
return []
|
|
kw_extractor = yake.KeywordExtractor(lan="en", n=2, top=max_keywords)
|
|
keywords = kw_extractor.extract_keywords(abstract)
|
|
genres = []
|
|
seen = set()
|
|
for kw, score in keywords:
|
|
kw_lower = kw.lower().strip()
|
|
if kw_lower in seen or kw_lower in _DROP_PHRASES:
|
|
continue
|
|
words = [w for w in kw_lower.split() if w not in _STOPWORDS]
|
|
cleaned = " ".join(words)
|
|
if not cleaned or len(cleaned) < 3 or cleaned in seen:
|
|
continue
|
|
if cleaned in _DROP_PHRASES:
|
|
continue
|
|
seen.add(cleaned)
|
|
genres.append(cleaned)
|
|
return genres
|
|
|
|
|
|
def lookup_paper_from_crossref(doi: str) -> dict:
|
|
url = CROSSREF_WORK_URL.format(doi)
|
|
headers = {"User-Agent": "Vrobbler/1.0 (mailto:hello@example.com)"}
|
|
response = requests.get(url, headers=headers)
|
|
|
|
if response.status_code != 200:
|
|
logger.warning(
|
|
"Bad response from Crossref",
|
|
extra={"doi": doi, "status": response.status_code},
|
|
)
|
|
return {"doi_id": doi}
|
|
|
|
try:
|
|
data = response.json()
|
|
except json.JSONDecodeError:
|
|
return {"doi_id": doi}
|
|
|
|
msg = data.get("message", {})
|
|
if not msg:
|
|
return {"doi_id": doi}
|
|
|
|
paper_dict = {"doi_id": doi}
|
|
|
|
titles = msg.get("title", [])
|
|
if titles:
|
|
paper_dict["title"] = titles[0]
|
|
|
|
abstract = msg.get("abstract", "")
|
|
if abstract:
|
|
stripped = _strip_jats(abstract)
|
|
paper_dict["abstract"] = stripped
|
|
genres = _extract_genres_from_abstract(stripped)
|
|
if genres:
|
|
paper_dict["genres"] = genres
|
|
|
|
author_dicts = []
|
|
for author in msg.get("author", []):
|
|
given = author.get("given", "")
|
|
family = author.get("family", "")
|
|
name = f"{given} {family}".strip()
|
|
if not name:
|
|
continue
|
|
entry = {"name": name}
|
|
orcid = author.get("ORCID", "")
|
|
if orcid:
|
|
orcid_id = orcid.replace("https://orcid.org/", "")
|
|
entry["authorId"] = orcid_id
|
|
author_dicts.append(entry)
|
|
if author_dicts:
|
|
paper_dict["author_dicts"] = author_dicts
|
|
|
|
container = msg.get("container-title", [])
|
|
if container:
|
|
paper_dict["journal_name"] = container[0]
|
|
|
|
volume = msg.get("volume")
|
|
if volume:
|
|
paper_dict["journal_volume"] = volume
|
|
|
|
page = msg.get("page")
|
|
if page:
|
|
try:
|
|
parts = page.split("-")
|
|
if len(parts) == 2:
|
|
paper_dict["pages"] = int(parts[1]) - int(parts[0])
|
|
except (ValueError, IndexError):
|
|
pass
|
|
|
|
for date_field in ("published-print", "published-online", "created"):
|
|
date_data = msg.get(date_field)
|
|
if date_data and date_data.get("date-parts"):
|
|
parts = date_data["date-parts"][0]
|
|
if len(parts) >= 1:
|
|
paper_dict["first_publish_year"] = int(parts[0])
|
|
if len(parts) >= 3:
|
|
paper_dict["publish_date"] = f"{parts[0]:04d}-{parts[1]:02d}-{parts[2]:02d}"
|
|
break
|
|
|
|
return paper_dict
|