Something went wrong. Try again.
Scripts to use scopus API to download economics journals and articles metadata + use of Elsevier API for full-text
Something went wrong. Try again.
14 kB · 427 lines
R
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428#: Extracting and cleaning Elsevier full-text XMLs -----------------------# Script goals:# * Clean and extract paragraph-level text from Elsevier full-text XML files.# * Save per-journal paragraph tables and update the article metadata with a# `full_text` flag indicating which articles have extracted full text.## Setup Instructions:# * Ensure `scripts/paths_and_packages.R` and `scripts/functions.R` are present# and contain the project paths and helper functions used here.# * Required packages are loaded by `paths_and_packages.R`; this script uses# data.table, stringr, cli, ggplot2, zoo, and purrr (via functions.R).# * Ensure the directory with Elsevier XML fulltexts exists at `xml_path`# (default: "~/data/elsevier/fulltexts_xml/") or adjust `xml_path`.## How it works (high level):# 1. Load project paths, package loader and helper functions.# 2. Load article metadata and annotate it with XML file sizes.# 3. (Interactive) Visual checks of file-size distribution and coverage by year.# 4. Select candidate XMLs to process (based on a minimum byte size).# 5. For each journal: extract paragraphs (using `extract_elsevier_fulltext`),# clean and label paragraphs, classify paragraph types, and save journal-level RDS.# 6. Update metadata `all_articles` marking articles with extracted full text.## ! Safety: This script expects `extract_elsevier_fulltext()` to exist in# `scripts/functions.R` and return a data.table-like list of paragraph rows.
#: Setup & safety checks ---------source("scripts/paths_and_packages.R") # defines `data_path`, `xml_data_path`, `intermediate_data_path`, etc.source("scripts/functions.R") # defines `extract_elsevier_fulltext()` etc.
# Load any package used only for a single helper (zoo used for na.locf).p_load(zoo)
# Directory holding the XML fulltexts — use path set in paths_and_packages.R# ! Stop early if XML directory is missing.if (!dir.exists(path.expand(elsevier_xml_path))) { stop( "Directory with fulltext XMLs not found: ", elsevier_xml_path, "\nPlease download XMLs or check `elsevier_xml_path` in paths_and_packages.R." )}
#: Load article metadata ---------# Load saved metadata containing `file_name`, `prism_doi`, `dc_identifier`, `full_text`, etc.con <- dbConnect( duckdb::duckdb(), dbdir = file.path(data_path, "scopus.duckdb"))
elsevier_issns <- read_rds(file.path( data_path, "intermediate_data", "elsevier_economics_journals.rds")) |> as.data.table() |> select(prism_issn, prism_e_issn) |> pivot_longer(everything(), values_to = "issn") |> pull(issn) %>% str_remove("-") %>% unique() %>% na.omit()
elsevier_articles <- tbl(con, "articles") %>% filter(prism_issn %in% elsevier_issns) %>% collect() %>% as.data.table()
# Add file size (bytes) for each article's XML file. Uses xml_data_path from paths.elsevier_articles[, file_size := file.info(file.path(elsevier_xml_path, file_name))$size]
#: Visual inspection of file sizes--------------------------------------# These plots are intended for exploratory work and are wrapped in `interactive()`.if (interactive()) { elsevier_articles |> filter(!is.na(file_size)) |> mutate(file_size_mo = file_size / 1e3) |> ggplot(aes(x = file_size_mo)) + geom_histogram(binwidth = 0.01) + labs( title = "Distribution of Elsevier XML file sizes", x = "File size (Kilo octets, log scale)", y = "Count" ) + scale_x_log10()
# Optional: quick time-series plot of availability (interactive only)--------- elsevier_articles |> filter(!is.na(file_size)) |> mutate( year = as.integer(str_sub(prism_cover_date, 1, 4)), full_text_likely = file_size > 50e3 ) |> count(full_text_likely, year) |> ggplot(aes(x = year, y = n, fill = full_text_likely)) + geom_col() + theme_minimal(base_size = 16) + labs( title = "Number of EER XML files with full text available", x = NULL, y = "Number of Articles" )}
#: Identify XML files to process-------------------------------------------min_size_bytes <- 12e3 # 12 KB, following visual inspection above
# Prepare list of XMLs to process.if ("elsevier_full_text" %in% colnames(elsevier_articles)) { cli_alert_info( "Removing from xml_to_process articles already marked as having full text" ) xml_to_process <- elsevier_articles[ file_size > min_size_bytes & !is.na(file_size) & elsevier_full_text == FALSE & subtype_description == "Article", .(prism_publication_name, file_name) ]} else { # Initialize full_text column and select candidate files elsevier_articles[, elsevier_full_text := FALSE] xml_to_process <- elsevier_articles[ file_size > min_size_bytes & !is.na(file_size), .(prism_publication_name, file_name) ]}
#: Prepare extraction parameters ------------------------------------------# Setting in advance sections to remove (just headers alone)section_to_remove <- c( "Keywords", "JEL classification", "References")
# XML namespaces (kept for reference / if helpers rely on them)ns <- c( ce = "http://www.elsevier.com/xml/common/dtd", dc = "http://purl.org/dc/elements/1.1/", xocs = "http://www.elsevier.com/xml/xocs/dtd", prism = "http://prismstandard.org/namespaces/basic/2.0/")
#: Identify journals already processed (temporary save directory) ----------processed_journals <- list.files( file.path(intermediate_data_path, "fulltexts_journal_rds"), pattern = "^elsevier_fulltext_extracted_.*\\.rds$") %>% str_remove("^elsevier_fulltext_extracted_") %>% str_remove("\\.rds$") %>% str_replace_all("_", " ")
# Unique journals in the to-process listjournals <- unique(xml_to_process$prism_publication_name)
#: Loop over journals and extract paragraphs --------------------------------for (journal in journals) { cli_alert_info("Starting processing for {journal}")
# Determine which files need processing for this journal. If an RDS already exists, # we'll only process new files and later combine them. if (journal %in% processed_journals) { cli_alert_info("Data for {journal} already processed. Loading the file") journal_old_data <- read_rds(file.path( intermediate_data_path, "fulltexts_journal_rds", str_c( "elsevier_fulltext_extracted_", str_replace_all(journal, "\\s+", "_"), ".rds" ) )) # If data already processed, it means that we have already processed some full text for this journal, # so we only want to process the remaining files journal_files <- xml_to_process[ prism_publication_name == journal & file_name %in% xml_to_process$file_name ]$file_name cli_alert_info( "Number of new files to process for {journal}: {length(journal_files)}" ) } else { journal_files <- xml_to_process[prism_publication_name == journal]$file_name }
# Map extraction function over files. `extract_elsevier_fulltext()` should # return a (list) of data.tables or NULL for missing/unparseable files. journal_text_data <- map( journal_files, ~ extract_elsevier_fulltext(xml_file = .x, xml_dir = elsevier_xml_path), .progress = TRUE )
# Combine results into a single data.table; rbindlist will drop NULLs by default. journal_text_data <- data.table::rbindlist(journal_text_data)
#Clean and label paragraphs (propagate section titles etc.) if (nrow(journal_text_data) > 0) { # Mark section-title nodes as section headings for downstream propagation journal_text_data[xml_name == "section-title", section := xml_text] journal_text_data_filtered <- journal_text_data[xml_text != ""] # allows notably to manage empty sections
#* Remove unwanted short sections (headers often include these labels) #* Important to keep it here, in case there is not introduction section name. journal_text_data_filtered <- journal_text_data_filtered[ !section %in% section_to_remove, ]
# Propagate section titles downward per document so each paragraph knows its section and remove sectin-title rows journal_text_data_filtered[, section := zoo::na.locf(section, na.rm = FALSE), by = file_name ] journal_text_data_filtered <- journal_text_data_filtered[ xml_name != "section-title", ]
#* Classify paragraphs and clean whitespace journal_text_data_filtered[ section %like% "(A|a)ppendi|Supplementary", type := "references_or_appendices" ] journal_text_data_filtered[ section %like% "^(Declaration|Disclosure).*interests?$|Statement$|interest statement$|conflict of interest|(A|a)cknowledge?ments?|^Fundings?$", type := "acknowledgments" ]
# Taking first into account text without any sections journal_text_data_filtered[, section := if (all(is.na(section))) "no_section", by = "file_name" ]
# Then, dealing with text with sections, but sometimes we have paragraphs without any section, often before the first section journal_text_data_filtered[ is.na(section) & is.na(type), type := "non-identified_section" ]
# We consider note-para as footnotes but after attributing "non-identified_section" to footnotes without section. # Indeed, most of the time, it is acknowledgments or unnecessary information. journal_text_data_filtered[ xml_name == "note-para" & is.na(type), type := "footnote" ]
# If the type of text is still NA, then we assume it is main text journal_text_data_filtered[is.na(type), type := "main_text"] journal_text_data_filtered <- journal_text_data_filtered[, text := str_squish(xml_text) ]
# We need to create an idea per document journal_text_data_filtered <- journal_text_data_filtered[, paragraph_id := seq_len(.N), by = "file_name" ]
# We can recover the scopus id and we don't need the file name anymore journal_text_data_filtered[, scopus_id := str_remove(file_name, "\\.xml$") |> str_replace("ID_", "ID:") ]
# We just keep the necessary columns. We remove the xml extractions. journal_text_data_filtered <- journal_text_data_filtered[, .(scopus_id, paragraph_id, text, section, type) ]
if (journal %in% processed_journals) { journal_text_data_filtered <- rbindlist( list(journal_old_data, journal_text_data_filtered), fill = TRUE ) |> unique() } } else { cli_alert_warning( "No new full text extracted for {journal}. Skipping to next journal." ) next }
# Save the extracted data saveRDS( journal_text_data_filtered, file.path( intermediate_data_path, "fulltexts_journal_rds", str_c( "elsevier_fulltext_extracted_", str_replace_all(journal, "\\s+", "_"), ".rds" ) ) )
# Remove large object from memory with silent warnings (in case journal_old_data doesn't exist) suppressWarnings(rm( journal_old_data, journal_text_data, journal_text_data_filtered )) gc()}
#: Update metadata `full_text` flag and save full texts in duckdb -----------final_processed_journals <- list.files( file.path(intermediate_data_path, "fulltexts_journal_rds"), pattern = "^elsevier_fulltext_extracted_.*\\.rds$") |> str_remove("^elsevier_fulltext_extracted_") |> str_remove("\\.rds$") |> str_replace_all("_", " ")
# Ensure required DB objects exist ---------------------------------------if (!"elsevier_full_text" %in% dbListFields(con, "articles")) { dbExecute( con, "ALTER TABLE articles ADD COLUMN elsevier_full_text BOOLEAN DEFAULT FALSE" )}
dbExecute( con, "CREATE TABLE IF NOT EXISTS full_text ( scopus_id VARCHAR, paragraph_id INTEGER, text VARCHAR, section VARCHAR, type VARCHAR )")
# Load each journal .rds, upsert into full_text, update metadatafor (journal in final_processed_journals) { cli_alert_info("Syncing journal data to duckdb: {journal}")
journal_data <- read_rds(file.path( intermediate_data_path, "fulltexts_journal_rds", str_c( "elsevier_fulltext_extracted_", str_replace_all(journal, "\\s+", "_"), ".rds" ) )) %>% as.data.table() %>% .[, .(scopus_id, paragraph_id, text, section, type)] %>% unique()
if (nrow(journal_data) == 0) { next }
scopus_ids <- unique(journal_data[, .(scopus_id)])
# Temp table of ids for idempotent delete+append and metadata update dbWriteTable( con, name = "tmp_scopus_ids", value = scopus_ids, temporary = TRUE, overwrite = TRUE )
# Avoid duplicates when script is re-run dbExecute( con, "DELETE FROM full_text WHERE scopus_id IN (SELECT scopus_id FROM tmp_scopus_ids)" )
dbWriteTable( con, name = "full_text", value = journal_data, append = TRUE )
# Update metadata flag for articles with extracted full text (dbplyr-friendly) rows_update( x = tbl(con, "articles"), y = tbl(con, "tmp_scopus_ids") |> transmute( scopus_id, elsevier_full_text = TRUE ), by = "scopus_id", unmatched = "ignore", in_place = TRUE )
dbExecute(con, "DROP TABLE IF EXISTS tmp_scopus_ids")
# Clean up memory rm(journal_data, scopus_ids) gc()}
#: Final interactive visual inspection (optional) ------------------------if (interactive()) { updated_elsevier_articles <- tbl(con, "articles") %>% filter(prism_issn %in% elsevier_issns) %>% collect() %>% as.data.table()
updated_elsevier_articles |> filter(!is.na(file_size)) |> mutate( year = as.integer(str_sub(prism_cover_date, 1, 4)), full_text_likely = file_size > 50e3 ) |> count(full_text_likely, year) |> ggplot(aes(x = year, y = n, fill = full_text_likely)) + geom_col() + theme_minimal(base_size = 16) + labs( title = "Number of EER XML files with full text available", x = NULL, y = "Number of Articles" )}