98 lines
3.5 KiB
Python
98 lines
3.5 KiB
Python
import json
|
|
import logging
|
|
from datetime import datetime
|
|
|
|
import requests
|
|
from django.conf import settings
|
|
|
|
PAPER_SEARCH_URL = (
|
|
"https://api.semanticscholar.org/graph/v1/paper/search/match?query={}"
|
|
)
|
|
PAPER_DETAIL_URL = "https://api.semanticscholar.org/graph/v1/paper/{}?fields=title,authors,url,year,abstract,externalIds,citationCount,referenceCount,journal,fieldsOfStudy,publicationDate,openAccessPdf"
|
|
PAPER_DOI_URL = "https://api.semanticscholar.org/graph/v1/paper/DOI:{}?fields=title,authors,url,year,abstract,externalIds,citationCount,referenceCount,journal,fieldsOfStudy,publicationDate,openAccessPdf"
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def get_api_result(url):
|
|
headers = {"User-Agent": "Vrobbler 0.11.12"}
|
|
response = requests.get(url, headers=headers)
|
|
|
|
if response.status_code != 200:
|
|
logger.warning("Bad response from Semantic", extra={"response": response})
|
|
return None
|
|
|
|
return response
|
|
|
|
|
|
def lookup_paper_from_semantic(title: str) -> dict:
|
|
paper_dict = {"title": title}
|
|
|
|
response = get_api_result(PAPER_SEARCH_URL.format(title))
|
|
if not response:
|
|
return paper_dict
|
|
|
|
semantic_id = json.loads(response.content).get("data")[0].get("paperId")
|
|
response = get_api_result(PAPER_DETAIL_URL.format(semantic_id))
|
|
|
|
result = json.loads(response.content)
|
|
|
|
if not result:
|
|
return paper_dict
|
|
|
|
paper_dict.update(_parse_semantic_result(result))
|
|
paper_dict.setdefault("title", title)
|
|
if paper_dict.get("publish_date"):
|
|
paper_dict["publish_date"] = datetime.strptime(
|
|
paper_dict["publish_date"], "%Y-%m-%d"
|
|
)
|
|
|
|
return paper_dict
|
|
|
|
|
|
def _parse_semantic_result(result: dict) -> dict:
|
|
paper_dict = {}
|
|
page_str = result.get("journal", {}).get("pages")
|
|
if page_str:
|
|
try:
|
|
start_page = page_str.split(" - ")[0]
|
|
end_page = page_str.split(" - ")[1]
|
|
paper_dict["pages"] = int(end_page) - int(start_page)
|
|
except IndexError:
|
|
pass
|
|
|
|
paper_dict["semantic_id"] = result.get("paperId")
|
|
paper_dict["doi_id"] = result.get("externalIds", {}).get("DOI")
|
|
paper_dict["arxiv_id"] = result.get("externalIds", {}).get("ArXiv")
|
|
paper_dict["pubmed_id"] = result.get("externalIds", {}).get("PubMed")
|
|
paper_dict["corpus_id"] = result.get("externalIds", {}).get("CorpusId")
|
|
paper_dict["semantic_title"] = result.get("title")
|
|
paper_dict["first_publish_year"] = result.get("year")
|
|
paper_dict["publish_date"] = result.get("publicationDate")
|
|
paper_dict["abstract"] = result.get("abstract")
|
|
paper_dict["tldr"] = result.get("bib", {}).get("abstract")
|
|
paper_dict["journal_name"] = result.get("journal", {}).get("name")
|
|
paper_dict["journal_volume"] = result.get("journal", {}).get("volume")
|
|
paper_dict["openaccess_pdf_url"] = result.get("openAccessPdf", {}).get("url")
|
|
paper_dict["base_run_time_seconds"] = paper_dict.get("pages", 10) * getattr(
|
|
settings, "AVERAGE_PAGE_READING_SECONDS", 60
|
|
)
|
|
paper_dict["author_dicts"] = result.get("authors")
|
|
paper_dict["genres"] = result.get("fieldsOfStudy")
|
|
return paper_dict
|
|
|
|
|
|
def lookup_paper_from_semantic_by_doi(doi: str) -> dict:
|
|
response = get_api_result(PAPER_DOI_URL.format(doi))
|
|
if not response:
|
|
return {"doi_id": doi}
|
|
|
|
result = json.loads(response.content)
|
|
if not result:
|
|
return {"doi_id": doi}
|
|
|
|
paper_dict = _parse_semantic_result(result)
|
|
if not paper_dict.get("title"):
|
|
paper_dict["title"] = result.get("title", f"Paper {doi}")
|
|
return paper_dict
|