#: Extracting and cleaning Elsevier full-text XMLs -----------------------
# Script goals:
# * Clean and extract paragraph-level text from Elsevier full-text XML files.
# * Save per-journal paragraph tables and update the article metadata with a
#   `full_text` flag indicating which articles have extracted full text.
#
# Setup Instructions:
# * Ensure `scripts/paths_and_packages.R` and `scripts/functions.R` are present
#   and contain the project paths and helper functions used here.
# * Required packages are loaded by `paths_and_packages.R`; this script uses
#   data.table, stringr, cli, ggplot2, zoo, and purrr (via functions.R).
# * Ensure the directory with Elsevier XML fulltexts exists at `xml_path`
#   (default: "~/data/elsevier/fulltexts_xml/") or adjust `xml_path`.
#
# How it works (high level):
# 1. Load project paths, package loader and helper functions.
# 2. Load article metadata and annotate it with XML file sizes.
# 3. (Interactive) Visual checks of file-size distribution and coverage by year.
# 4. Select candidate XMLs to process (based on a minimum byte size).
# 5. For each journal: extract paragraphs (using `extract_elsevier_fulltext`),
#    clean and label paragraphs, classify paragraph types, and save journal-level RDS.
# 6. Update metadata `all_articles` marking articles with extracted full text.
#
# ! Safety: This script expects `extract_elsevier_fulltext()` to exist in
#   `scripts/functions.R` and return a data.table-like list of paragraph rows.

#: Setup & safety checks ---------
source("scripts/paths_and_packages.R") # defines `data_path`, `xml_data_path`, `intermediate_data_path`, etc.
source("scripts/functions.R") # defines `extract_elsevier_fulltext()` etc.

# Load any package used only for a single helper (zoo used for na.locf).
p_load(zoo)

# Directory holding the XML fulltexts — use path set in paths_and_packages.R
# ! Stop early if XML directory is missing.
if (!dir.exists(path.expand(elsevier_xml_path))) {
  stop(
    "Directory with fulltext XMLs not found: ",
    elsevier_xml_path,
    "\nPlease download XMLs or check `elsevier_xml_path` in paths_and_packages.R."
  )
}

#: Load article metadata ---------
# Load saved metadata containing `file_name`, `prism_doi`, `dc_identifier`, `full_text`, etc.
con <- dbConnect(
  duckdb::duckdb(),
  dbdir = file.path(data_path, "scopus.duckdb")
)

elsevier_issns <- read_rds(file.path(
  data_path,
  "intermediate_data",
  "elsevier_economics_journals.rds"
)) |>
  as.data.table() |>
  select(prism_issn, prism_e_issn) |>
  pivot_longer(everything(), values_to = "issn") |>
  pull(issn) %>%
  str_remove("-") %>%
  unique() %>%
  na.omit()

elsevier_articles <- tbl(con, "articles") %>%
  filter(prism_issn %in% elsevier_issns) %>%
  collect() %>%
  as.data.table()


# Add file size (bytes) for each article's XML file. Uses xml_data_path from paths.
elsevier_articles[,
  file_size := file.info(file.path(elsevier_xml_path, file_name))$size
]

#: Visual inspection of file sizes--------------------------------------
# These plots are intended for exploratory work and are wrapped in `interactive()`.
if (interactive()) {
  elsevier_articles |>
    filter(!is.na(file_size)) |>
    mutate(file_size_mo = file_size / 1e3) |>
    ggplot(aes(x = file_size_mo)) +
    geom_histogram(binwidth = 0.01) +
    labs(
      title = "Distribution of Elsevier XML file sizes",
      x = "File size (Kilo octets, log scale)",
      y = "Count"
    ) +
    scale_x_log10()

  # Optional: quick time-series plot of availability (interactive only)---------
  elsevier_articles |>
    filter(!is.na(file_size)) |>
    mutate(
      year = as.integer(str_sub(prism_cover_date, 1, 4)),
      full_text_likely = file_size > 50e3
    ) |>
    count(full_text_likely, year) |>
    ggplot(aes(x = year, y = n, fill = full_text_likely)) +
    geom_col() +
    theme_minimal(base_size = 16) +
    labs(
      title = "Number of EER XML files with full text available",
      x = NULL,
      y = "Number of Articles"
    )
}

#: Identify XML files to process-------------------------------------------
min_size_bytes <- 12e3 # 12 KB, following visual inspection above

# Prepare list of XMLs to process.
if ("elsevier_full_text" %in% colnames(elsevier_articles)) {
  cli_alert_info(
    "Removing from xml_to_process articles already marked as having full text"
  )
  xml_to_process <- elsevier_articles[
    file_size > min_size_bytes &
      !is.na(file_size) &
      elsevier_full_text == FALSE &
      subtype_description == "Article",
    .(prism_publication_name, file_name)
  ]
} else {
  # Initialize full_text column and select candidate files
  elsevier_articles[, elsevier_full_text := FALSE]
  xml_to_process <- elsevier_articles[
    file_size > min_size_bytes & !is.na(file_size),
    .(prism_publication_name, file_name)
  ]
}

#: Prepare extraction parameters ------------------------------------------
# Setting in advance sections to remove (just headers alone)
section_to_remove <- c(
  "Keywords",
  "JEL classification",
  "References"
)

# XML namespaces (kept for reference / if helpers rely on them)
ns <- c(
  ce = "http://www.elsevier.com/xml/common/dtd",
  dc = "http://purl.org/dc/elements/1.1/",
  xocs = "http://www.elsevier.com/xml/xocs/dtd",
  prism = "http://prismstandard.org/namespaces/basic/2.0/"
)

#: Identify journals already processed (temporary save directory) ----------
processed_journals <- list.files(
  file.path(intermediate_data_path, "fulltexts_journal_rds"),
  pattern = "^elsevier_fulltext_extracted_.*\\.rds$"
) %>%
  str_remove("^elsevier_fulltext_extracted_") %>%
  str_remove("\\.rds$") %>%
  str_replace_all("_", " ")

# Unique journals in the to-process list
journals <- unique(xml_to_process$prism_publication_name)

#: Loop over journals and extract paragraphs --------------------------------
for (journal in journals) {
  cli_alert_info("Starting processing for {journal}")

  # Determine which files need processing for this journal. If an RDS already exists,
  # we'll only process new files and later combine them.
  if (journal %in% processed_journals) {
    cli_alert_info("Data for {journal} already processed. Loading the file")
    journal_old_data <- read_rds(file.path(
      intermediate_data_path,
      "fulltexts_journal_rds",
      str_c(
        "elsevier_fulltext_extracted_",
        str_replace_all(journal, "\\s+", "_"),
        ".rds"
      )
    ))
    # If data already processed, it means that we have already processed some full text for this journal,
    # so we only want to process the remaining files
    journal_files <- xml_to_process[
      prism_publication_name == journal &
        file_name %in% xml_to_process$file_name
    ]$file_name
    cli_alert_info(
      "Number of new files to process for {journal}: {length(journal_files)}"
    )
  } else {
    journal_files <- xml_to_process[prism_publication_name == journal]$file_name
  }

  # Map extraction function over files. `extract_elsevier_fulltext()` should
  # return a (list) of data.tables or NULL for missing/unparseable files.
  journal_text_data <- map(
    journal_files,
    ~ extract_elsevier_fulltext(xml_file = .x, xml_dir = elsevier_xml_path),
    .progress = TRUE
  )

  # Combine results into a single data.table; rbindlist will drop NULLs by default.
  journal_text_data <- data.table::rbindlist(journal_text_data)

  #Clean and label paragraphs (propagate section titles etc.)
  if (nrow(journal_text_data) > 0) {
    # Mark section-title nodes as section headings for downstream propagation
    journal_text_data[xml_name == "section-title", section := xml_text]
    journal_text_data_filtered <- journal_text_data[xml_text != ""] # allows notably to manage empty sections

    #* Remove unwanted short sections (headers often include these labels)
    #* Important to keep it here, in case there is not introduction section name.
    journal_text_data_filtered <- journal_text_data_filtered[
      !section %in% section_to_remove,
    ]

    # Propagate section titles downward per document so each paragraph knows its section and remove sectin-title rows
    journal_text_data_filtered[,
      section := zoo::na.locf(section, na.rm = FALSE),
      by = file_name
    ]
    journal_text_data_filtered <- journal_text_data_filtered[
      xml_name != "section-title",
    ]

    #* Classify paragraphs and clean whitespace
    journal_text_data_filtered[
      section %like% "(A|a)ppendi|Supplementary",
      type := "references_or_appendices"
    ]
    journal_text_data_filtered[
      section %like%
        "^(Declaration|Disclosure).*interests?$|Statement$|interest statement$|conflict of interest|(A|a)cknowledge?ments?|^Fundings?$",
      type := "acknowledgments"
    ]

    # Taking first into account text without any sections
    journal_text_data_filtered[,
      section := if (all(is.na(section))) "no_section",
      by = "file_name"
    ]

    # Then, dealing with text with sections, but sometimes we have paragraphs without any section, often before the first section
    journal_text_data_filtered[
      is.na(section) & is.na(type),
      type := "non-identified_section"
    ]

    # We consider note-para as footnotes but after attributing "non-identified_section" to footnotes without section.
    # Indeed, most of the time, it is acknowledgments or unnecessary information.
    journal_text_data_filtered[
      xml_name == "note-para" & is.na(type),
      type := "footnote"
    ]

    # If the type of text is still NA, then we assume it is main text
    journal_text_data_filtered[is.na(type), type := "main_text"]
    journal_text_data_filtered <- journal_text_data_filtered[,
      text := str_squish(xml_text)
    ]

    # We need to create an idea per document
    journal_text_data_filtered <- journal_text_data_filtered[,
      paragraph_id := seq_len(.N),
      by = "file_name"
    ]

    # We can recover the scopus id and we don't need the file name anymore
    journal_text_data_filtered[,
      scopus_id := str_remove(file_name, "\\.xml$") |> str_replace("ID_", "ID:")
    ]

    # We just keep the necessary columns. We remove the xml extractions.
    journal_text_data_filtered <- journal_text_data_filtered[,
      .(scopus_id, paragraph_id, text, section, type)
    ]

    if (journal %in% processed_journals) {
      journal_text_data_filtered <- rbindlist(
        list(journal_old_data, journal_text_data_filtered),
        fill = TRUE
      ) |>
        unique()
    }
  } else {
    cli_alert_warning(
      "No new full text extracted for {journal}. Skipping to next journal."
    )
    next
  }

  # Save the extracted data
  saveRDS(
    journal_text_data_filtered,
    file.path(
      intermediate_data_path,
      "fulltexts_journal_rds",
      str_c(
        "elsevier_fulltext_extracted_",
        str_replace_all(journal, "\\s+", "_"),
        ".rds"
      )
    )
  )

  # Remove large object from memory with silent warnings (in case journal_old_data doesn't exist)
  suppressWarnings(rm(
    journal_old_data,
    journal_text_data,
    journal_text_data_filtered
  ))
  gc()
}

#: Update metadata `full_text` flag and save full texts in duckdb -----------
final_processed_journals <- list.files(
  file.path(intermediate_data_path, "fulltexts_journal_rds"),
  pattern = "^elsevier_fulltext_extracted_.*\\.rds$"
) |>
  str_remove("^elsevier_fulltext_extracted_") |>
  str_remove("\\.rds$") |>
  str_replace_all("_", " ")

# Ensure required DB objects exist ---------------------------------------
if (!"elsevier_full_text" %in% dbListFields(con, "articles")) {
  dbExecute(
    con,
    "ALTER TABLE articles ADD COLUMN elsevier_full_text BOOLEAN DEFAULT FALSE"
  )
}

dbExecute(
  con,
  "CREATE TABLE IF NOT EXISTS full_text (
     scopus_id VARCHAR,
     paragraph_id INTEGER,
     text VARCHAR,
     section VARCHAR,
     type VARCHAR
   )"
)

# Load each journal .rds, upsert into full_text, update metadata
for (journal in final_processed_journals) {
  cli_alert_info("Syncing journal data to duckdb: {journal}")

  journal_data <- read_rds(file.path(
    intermediate_data_path,
    "fulltexts_journal_rds",
    str_c(
      "elsevier_fulltext_extracted_",
      str_replace_all(journal, "\\s+", "_"),
      ".rds"
    )
  )) %>%
    as.data.table() %>%
    .[, .(scopus_id, paragraph_id, text, section, type)] %>%
    unique()

  if (nrow(journal_data) == 0) {
    next
  }

  scopus_ids <- unique(journal_data[, .(scopus_id)])

  # Temp table of ids for idempotent delete+append and metadata update
  dbWriteTable(
    con,
    name = "tmp_scopus_ids",
    value = scopus_ids,
    temporary = TRUE,
    overwrite = TRUE
  )

  # Avoid duplicates when script is re-run
  dbExecute(
    con,
    "DELETE FROM full_text
     WHERE scopus_id IN (SELECT scopus_id FROM tmp_scopus_ids)"
  )

  dbWriteTable(
    con,
    name = "full_text",
    value = journal_data,
    append = TRUE
  )

  # Update metadata flag for articles with extracted full text (dbplyr-friendly)
  rows_update(
    x = tbl(con, "articles"),
    y = tbl(con, "tmp_scopus_ids") |>
      transmute(
        scopus_id,
        elsevier_full_text = TRUE
      ),
    by = "scopus_id",
    unmatched = "ignore",
    in_place = TRUE
  )

  dbExecute(con, "DROP TABLE IF EXISTS tmp_scopus_ids")

  # Clean up memory
  rm(journal_data, scopus_ids)
  gc()
}

#: Final interactive visual inspection (optional) ------------------------
if (interactive()) {
  updated_elsevier_articles <- tbl(con, "articles") %>%
    filter(prism_issn %in% elsevier_issns) %>%
    collect() %>%
    as.data.table()

  updated_elsevier_articles |>
    filter(!is.na(file_size)) |>
    mutate(
      year = as.integer(str_sub(prism_cover_date, 1, 4)),
      full_text_likely = file_size > 50e3
    ) |>
    count(full_text_likely, year) |>
    ggplot(aes(x = year, y = n, fill = full_text_likely)) +
    geom_col() +
    theme_minimal(base_size = 16) +
    labs(
      title = "Number of EER XML files with full text available",
      x = NULL,
      y = "Number of Articles"
    )
}