From 549678dcdd2e230be29f84e7ce1e034ccd8a96c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Sch=C3=BCtt?= Date: Fri, 12 Jul 2024 23:49:19 +0200 Subject: [PATCH] Docs: Move bibliography formatting logic to Python --- docs/README.md | 7 +- docs/generate_input_reference.py | 75 +++++---- src/common/bibliography.F | 3 +- src/common/reference_manager.F | 250 ++++++++-------------------- src/environment.F | 9 +- src/start/cp2k_runs.F | 16 +- tools/docker/scripts/test_manual.sh | 2 +- 7 files changed, 118 insertions(+), 244 deletions(-) diff --git a/docs/README.md b/docs/README.md index a382cfd008..da7a3cbf85 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,17 +18,16 @@ To build a local version of the manual perform the following steps: pip3 install -r ./requirements.txt ``` -1. (optional) Build a CP2K binary and use it to generate the `cp2k_input.xml` and `references.html` - files: +1. (optional) Build a CP2K binary and use it to generate the `cp2k_input.xml` file: ``` ../exe/local/cp2k.psmp --xml ``` -1. (optional) Generate Markdown pages from the aforementioned files: +1. (optional) Generate Markdown pages from the `cp2k_input.xml` file: ``` - ./generate_input_reference.py ./cp2k_input.xml ./references.html + ./generate_input_reference.py ./cp2k_input.xml ``` 1. Run Sphinx: diff --git a/docs/generate_input_reference.py b/docs/generate_input_reference.py index 73ccff30bc..cea67d97cb 100755 --- a/docs/generate_input_reference.py +++ b/docs/generate_input_reference.py @@ -7,6 +7,8 @@ import xml.etree.ElementTree as ET from pathlib import Path import re import sys +import locale +from datetime import datetime from collections import defaultdict from functools import cache @@ -16,62 +18,63 @@ SectionPath = Tuple[str, ...] # ====================================================================================== def main() -> None: - if len(sys.argv) != 3: - print("generate_input_reference.py ") + if len(sys.argv) != 2: + print("generate_input_reference.py ") sys.exit(1) - cp2k_input_xml_fn, references_html_fn = sys.argv[1:] + cp2k_input_xml_fn = sys.argv[1] output_dir = Path(__file__).resolve().parent - - build_bibliography(references_html_fn, output_dir) - build_input_reference(cp2k_input_xml_fn, output_dir) + locale.setlocale(locale.LC_ALL, locale="C.UTF8") # needed for strptime + root = ET.parse(cp2k_input_xml_fn).getroot() + build_bibliography(root, output_dir) + build_input_reference(root, output_dir) # ====================================================================================== -def build_bibliography(references_html_fn: str, output_dir: Path) -> None: - content = Path(references_html_fn).read_text() - entries = re.findall(r".*?", content, re.DOTALL) +def get_reference_epoch(reference: ET.Element) -> int: + year = int(get_text(reference.find("YEAR")) or "1900") + day = int(get_text(reference.find("DAY")) or "0") + month_str = get_text(reference.find("MONTH")) + month = datetime.strptime(month_str, "%b").month if month_str else 0 + return day + 31 * month + 12 * 31 * (year - 1900) + + +# ====================================================================================== +def build_bibliography(root: ET.Element, output_dir: Path) -> None: + references = root.findall("REFERENCE") + references.sort(key=get_reference_epoch, reverse=True) output = [] output += ["%", "% This file was created by generate_input_reference.py", "%"] output += ["# Bibliography", ""] - for entry in entries: - pattern = ( - r'\[(.*?)\]\n (.*?)
' - r"(.*?)" - ) - parts = re.search(pattern, entry, re.DOTALL) - assert parts - key = parts.group(1) - title = parts.group(3).strip() - if "
" in parts.group(2): - m = re.match(r"(.*?)
(.*)", parts.group(2), re.DOTALL) - assert m - authors, mix = m.groups() - else: - authors, mix = "", parts.group(2) - - if "https://doi.org" in mix: - m = re.match(r'\s*(.*?)', mix, re.DOTALL) - assert m - doi, ref = m.groups() - else: - doi, ref = "", mix.strip() + for r in references: + key = r.attrib.get("key") + authors = "; ".join([get_text(a) for a in r.findall("AUTHOR")]) + doi = get_text(r.find("DOI")) + source = get_text(r.find("SOURCE")) + volume = get_text(r.find("VOLUME")) + issue = get_text(r.find("ISSUE")) + pages = get_text(r.find("PAGES")) + year = get_text(r.find("YEAR")) + title = get_text(r.find("TITLE")) + ref = source + ref += f", {volume}" if volume else "" + ref += f" ({issue})" if volume and issue else "" + ref += f", {pages}" + ref += f" ({year})." if year else "" output += [f"({key})=", f"## {key}", ""] if doi: - output += [f"{authors} **{title}** _[{ref}]({doi})_", ""] + output += [f"{authors}. **{title}.** _[{ref}](https://doi.org/{doi})_", ""] else: - output += [f"{authors} **{title}** _{ref}_", ""] + output += [f"{authors}. **{title}.** _{ref}_", ""] write_file(output_dir / "bibliography.md", "\n".join(output)) # ====================================================================================== -def build_input_reference(cp2k_input_xml_fn: str, output_dir: Path) -> None: - tree = ET.parse(cp2k_input_xml_fn) - root = tree.getroot() +def build_input_reference(root: ET.Element, output_dir: Path) -> None: num_files_written = process_section(root, ("CP2K_INPUT",), False, output_dir) # Build landing page. diff --git a/src/common/bibliography.F b/src/common/bibliography.F index 481ca415b9..4326613f52 100644 --- a/src/common/bibliography.F +++ b/src/common/bibliography.F @@ -3140,7 +3140,7 @@ CONTAINS "VL 25", & "IS 29", & "AR 295501", & - "PD JUL 24 2013", & + "PD JUL 24", & "PY 2013", & "TC 3", & "ZB 1", & @@ -3751,7 +3751,6 @@ CONTAINS "AU Messmer, Peter", & "AU Hutter, Juerg", & "AU VandeVondele, Joost", & - "VL John Wiley & Sons, Ltd", & "SN 9781118670712", & "UR https://doi.org/10.1002/9781118670712.ch8", & "DO 10.1002/9781118670712.ch8", & diff --git a/src/common/reference_manager.F b/src/common/reference_manager.F index 1d4cd7b1b0..684265fbad 100644 --- a/src/common/reference_manager.F +++ b/src/common/reference_manager.F @@ -24,18 +24,21 @@ MODULE reference_manager USE kinds, ONLY: default_string_length USE message_passing, ONLY: mp_para_env_type - USE string_utilities, ONLY: uppercase + USE string_utilities, ONLY: substitute_special_xml_tokens,& + uppercase USE util, ONLY: sort #include "../base/base_uses.f90" IMPLICIT NONE - PUBLIC :: print_reference, print_all_references, cite_reference - PUBLIC :: collect_citations_from_ranks + PUBLIC :: cite_reference + PUBLIC :: collect_citations_from_ranks + PUBLIC :: print_cited_references + PUBLIC :: export_references_as_xml - INTEGER, PUBLIC, PARAMETER :: print_format_isi = 101, & - print_format_journal = 102, & - print_format_html = 103 + PUBLIC :: add_reference ! use this one only in bibliography.F + PUBLIC :: remove_all_references ! use only in f77_interface.F + PUBLIC :: get_citation_key ! a string key describing the reference (e.g. Kohn1965b) PRIVATE @@ -49,7 +52,6 @@ MODULE reference_manager INTEGER, PARAMETER :: ISI_length = 128 ! the way we store a reference, should remain fully private -! ************************************************************************************************** TYPE reference_type PRIVATE ! the reference in a format as returned by the web of science @@ -63,19 +65,14 @@ MODULE reference_manager END TYPE reference_type ! useful to build arrays -! ************************************************************************************************** TYPE reference_p_type TYPE(reference_type), POINTER :: ref => NULL() END TYPE - ! thebibliography + ! the bibliography INTEGER, SAVE :: nbib = 0 TYPE(reference_p_type), DIMENSION(max_reference) :: thebib - PUBLIC :: add_reference, & ! use this one only in bibliography.F - remove_all_references, & ! use only in f77_interface.F - get_citation_key ! a string key describing the reference (e.g. Kohn1965b) - CONTAINS ! ************************************************************************************************** @@ -211,135 +208,38 @@ CONTAINS DEALLOCATE (thebib(i)%ref) END DO END SUBROUTINE remove_all_references -!****f* reference_manager/print_all_references * ! ************************************************************************************************** -!> \brief printout of all references in a specific format -!> optionally printing only those that are actually cited -!> during program execution -!> \param cited_only print only those marked as cited -!> \param sorted sort entries most recent first according to the date, -!> otherways sort with respect to key -!> \param FORMAT see module parameters print_format_XXXXXXXXX +!> \brief printout of all cited references in the journal format sorted by publication year !> \param unit ... -!> \param list optionally, output a sub-list only !> \par History -!> 08.2007 Joost VandeVondele [ ] +!> 08.2007 Joost VandeVondele +!> 07.2024 Ole Schuett ! ************************************************************************************************** - SUBROUTINE print_all_references(cited_only, sorted, FORMAT, unit, list) - LOGICAL, INTENT(IN) :: cited_only, sorted - INTEGER, INTENT(IN) :: FORMAT, unit - INTEGER, DIMENSION(:), INTENT(IN), OPTIONAL :: list + SUBROUTINE print_cited_references(unit) + INTEGER, INTENT(IN) :: unit - INTEGER :: I, irecord, nref - INTEGER, ALLOCATABLE, DIMENSION(:) :: indx, irank, ival + INTEGER :: i + INTEGER, ALLOCATABLE, DIMENSION(:) :: irank, ival -! we'll sort the references wrt to the publication year -! the most recent first, publications without a year get last + ALLOCATE (ival(nbib), irank(nbib)) - IF (PRESENT(list)) THEN - nref = SIZE(list) - ELSE - nref = nbib - END IF - - ALLOCATE (ival(nref)) - ALLOCATE (irank(nref)) - ALLOCATE (indx(nref)) - - IF (PRESENT(list)) THEN - indx(:) = list - ELSE - DO I = 1, nref - indx(I) = I - END DO - END IF - - DO I = 1, nref - irank(I) = I + ! we'll sort the references wrt to the publication year + ! the most recent first, publications without a year get last + DO i = 1, nbib + irank(i) = i + ival(i) = -get_epoch(thebib(i)%ref%ISI_record) END DO + CALL sort(ival, nbib, irank) - IF (sorted) THEN - DO I = 1, nref - ival(I) = -get_epoch(thebib(indx(I))%ref%ISI_record) - END DO - ELSE - DO I = 1, nref - ival(I) = indx(I) - END DO - END IF - CALL sort(ival, nref, irank) - - SELECT CASE (FORMAT) - CASE (print_format_isi) - CASE (print_format_journal) - WRITE (unit, '(A)') "" - CASE (print_format_html) - WRITE (unit, '(A)') '' - CASE DEFAULT - CPABORT("print_reference: wrong format") - END SELECT - - DO I = 1, nref - irecord = indx(irank(I)) - IF (.NOT. cited_only .OR. thebib(irecord)%ref%is_cited) THEN - SELECT CASE (FORMAT) - CASE (print_format_isi) - CASE (print_format_journal) - CASE (print_format_html) - WRITE (unit, '(A)') "' - CASE DEFAULT - CPABORT("print_reference: wrong format") - END SELECT + DO i = 1, nbib + IF (thebib(irank(i))%ref%is_cited) THEN + CALL print_reference_journal(key=irank(i), unit=unit) + WRITE (unit, '(A)') "" END IF END DO - IF (FORMAT .EQ. print_format_html) THEN - WRITE (unit, '(A)') "
"//'['//TRIM(thebib(irecord)%ref%citation_key)//']'//"" - CASE DEFAULT - CPABORT("print_reference: wrong format") - END SELECT - - CALL print_reference(irecord, FORMAT, unit) - - SELECT CASE (FORMAT) - CASE (print_format_isi) - CASE (print_format_journal) - WRITE (unit, '(A)') "" - CASE (print_format_html) - WRITE (unit, '(A)') '
" - END IF - END SUBROUTINE print_all_references -!****f* reference_manager/print_reference * - -! ************************************************************************************************** -!> \brief printout of a specified reference to a given unit in a selectable format -!> \param key as returned from add_reference -!> \param FORMAT see module parameters print_format_XXXXXXXXX -!> \param unit ... -!> \par History -!> 08.2007 Joost VandeVondele [ ] -! ************************************************************************************************** - SUBROUTINE print_reference(key, FORMAT, unit) - INTEGER, INTENT(IN) :: key, FORMAT, unit - - INTEGER :: I - - IF (key < 1 .OR. key > max_reference) CPABORT("citation key out of range") - - SELECT CASE (FORMAT) - CASE (print_format_isi) - DO I = 1, SIZE(thebib(key)%ref%ISI_record) - WRITE (unit, '(T2,A)') TRIM(thebib(key)%ref%ISI_record(I)) - END DO - CASE (print_format_journal) - CALL print_reference_journal(key, unit) - CASE (print_format_html) - CALL print_reference_html(key, unit) - CASE DEFAULT - CPABORT("print_reference: wrong format") - END SELECT - END SUBROUTINE print_reference + END SUBROUTINE print_cited_references ! ************************************************************************************************** !> \brief prints a reference in a journal style citation format, @@ -424,68 +324,52 @@ CONTAINS END SUBROUTINE print_reference_journal ! ************************************************************************************************** -!> \brief prints a reference in a journal style citation format, -!> adding 'beautifying' html tags, and a link to the journal -!> using the DOI -!> \param key ... +!> \brief Exports all references as XML. !> \param unit ... -!> \par History -!> 08.2007 created [Joost VandeVondele] +!> \author Ole Schuett ! ************************************************************************************************** - SUBROUTINE print_reference_html(key, unit) - INTEGER, INTENT(IN) :: key, unit + SUBROUTINE export_references_as_xml(unit) + INTEGER, INTENT(IN) :: unit CHARACTER(LEN=ISI_length) :: author, title - CHARACTER(LEN=ISI_length*4) :: journal - INTEGER :: iauthor, ititle, line + INTEGER :: i, line -! write the author list + DO i = 1, nbib + WRITE (unit, '(T2,A)') '' - WRITE (unit, '(T2,A,I0,A)', ADVANCE="NO") '' - line = 1; iauthor = 0 - author = get_next_author(thebib(key)%ref%ISI_record, line) - DO WHILE (author .NE. "") - iauthor = iauthor + 1 - IF (iauthor .NE. 1) WRITE (unit, '(A)', ADVANCE="NO") "; " - WRITE (unit, '(A)', ADVANCE="NO") TRIM(author) - author = get_next_author(thebib(key)%ref%ISI_record, line) + ! Authors + line = 1 + DO WHILE (.TRUE.) + author = get_next_author(thebib(i)%ref%ISI_record, line) + IF (LEN_TRIM(author) == 0) EXIT + WRITE (unit, '(T3,A)') ''//TRIM(author)//'' + END DO + + ! DOI, journal, volume (issue), pages (year). + WRITE (unit, '(T3,A)') ''//TRIM(substitute_special_xml_tokens(thebib(i)%ref%DOI))//'' + WRITE (unit, '(T3,A)') ''//TRIM(get_source(thebib(i)%ref%ISI_record))//'' + WRITE (unit, '(T3,A)') ''//TRIM(get_volume(thebib(i)%ref%ISI_record))//'' + WRITE (unit, '(T3,A)') ''//TRIM(get_issue(thebib(i)%ref%ISI_record))//'' + WRITE (unit, '(T3,A)') ''//TRIM(get_pages(thebib(i)%ref%ISI_record))//'' + WRITE (unit, '(T3,A)') ''//TRIM(get_year(thebib(i)%ref%ISI_record))//'' + WRITE (unit, '(T3,A)') ''//TRIM(get_month(thebib(i)%ref%ISI_record))//'' + WRITE (unit, '(T3,A)') ''//TRIM(ADJUSTL(get_day(thebib(i)%ref%ISI_record)))//'' + + ! Title + line = 1 + title = get_next_title(thebib(i)%ref%ISI_record, line) + WRITE (unit, '(T3,A)', ADVANCE='NO') ''//TRIM(title) + DO WHILE (.TRUE.) + title = ADJUSTL(get_next_title(thebib(i)%ref%ISI_record, line)) + IF (LEN_TRIM(title) == 0) EXIT + WRITE (unit, '(1X,A)', ADVANCE="NO") TRIM(title) + END DO + WRITE (unit, '(A)') '' + + WRITE (unit, '(T2,A)') '' END DO - IF (iauthor > 0) WRITE (unit, '(A)') ".
" - ! DOI - IF (thebib(key)%ref%DOI .NE. "") THEN - WRITE (unit, '(T2,A)', ADVANCE="NO") '' - END IF - ! Journal, volume (issue), pages (year). - journal = TRIM(get_source(thebib(key)%ref%ISI_record)) - IF (get_volume(thebib(key)%ref%ISI_record) .NE. "") THEN - journal = TRIM(journal)//", "//get_volume(thebib(key)%ref%ISI_record) - IF (get_issue(thebib(key)%ref%ISI_record) .NE. "") THEN - journal = TRIM(journal)//" ("//TRIM(get_issue(thebib(key)%ref%ISI_record))//")" - END IF - END IF - journal = TRIM(journal)//", "//get_pages(thebib(key)%ref%ISI_record) - IF (get_year(thebib(key)%ref%ISI_record) .NE. "") THEN - journal = TRIM(journal)//" ("//TRIM(get_year(thebib(key)%ref%ISI_record))//")." - END IF - WRITE (unit, '(A)', ADVANCE="NO") TRIM(journal) - IF (thebib(key)%ref%DOI .NE. "") THEN - WRITE (unit, '(A)', ADVANCE="NO") '' - END IF - WRITE (unit, '(A)') "
" - - ! Title - line = 1; ititle = 0 - title = get_next_title(thebib(key)%ref%ISI_record, line) - DO WHILE (title .NE. "") - ititle = ititle + 1 - IF (ititle .NE. 1) WRITE (unit, '(A)') "" - WRITE (unit, '(T2,A)', ADVANCE="NO") TRIM(title) - title = get_next_title(thebib(key)%ref%ISI_record, line) - END DO - IF (ititle > 0) WRITE (unit, '(A)') "." - - END SUBROUTINE print_reference_html + END SUBROUTINE export_references_as_xml ! ************************************************************************************************** !> \brief returns the corresponding fields from an ISI record. diff --git a/src/environment.F b/src/environment.F index 64e9f36476..f4a47fc345 100644 --- a/src/environment.F +++ b/src/environment.F @@ -101,8 +101,7 @@ MODULE environment write_rng_matrices USE physcon, ONLY: write_physcon USE reference_manager, ONLY: collect_citations_from_ranks,& - print_all_references,& - print_format_journal + print_cited_references USE string_utilities, ONLY: ascii_to_string,& integer_to_string,& string_to_ascii @@ -1257,13 +1256,11 @@ CONTAINS WRITE (UNIT=iw, FMT="(T2,A,T30,A,T80,A)") "-", "R E F E R E N C E S", "-" WRITE (UNIT=iw, FMT="(T2,A,T80,A)") "-", "-" WRITE (UNIT=iw, FMT="(T2,A)") REPEAT("-", 79) - WRITE (UNIT=iw, FMT="(T2,A)") "" WRITE (UNIT=iw, FMT="(T2,A)") TRIM(cp2k_version)//", the CP2K developers group ("//TRIM(cp2k_year)//")." WRITE (UNIT=iw, FMT="(T2,A)") "CP2K is freely available from "//TRIM(cp2k_home)//" ." - - CALL print_all_references(sorted=.TRUE., cited_only=.TRUE., & - FORMAT=print_format_journal, unit=iw) + WRITE (UNIT=iw, FMT="(T2,A)") "" + CALL print_cited_references(unit=iw) END IF CALL cp_print_key_finished_output(iw, logger, root_section, & "GLOBAL%REFERENCES") diff --git a/src/start/cp2k_runs.F b/src/start/cp2k_runs.F index 508e46a478..9784b6c66d 100644 --- a/src/start/cp2k_runs.F +++ b/src/start/cp2k_runs.F @@ -126,8 +126,7 @@ MODULE cp2k_runs USE qs_environment_types, ONLY: get_qs_env USE qs_linres_module, ONLY: linres_calculation USE qs_tddfpt_module, ONLY: tddfpt_calculation - USE reference_manager, ONLY: print_all_references,& - print_format_html + USE reference_manager, ONLY: export_references_as_xml USE rt_propagation, ONLY: rt_prop_setup USE sirius_interface, ONLY: cp_sirius_finalize,& cp_sirius_init @@ -913,6 +912,9 @@ CONTAINS " "//TRIM(cp2k_year)//"", & " "//TRIM(compile_date)//"", & " "//TRIM(compile_revision)//"" + + CALL export_references_as_xml(unit_number) + DO i = 1, root_section%n_subsections CALL write_section_xml(root_section%subsections(i)%section, 1, unit_number) END DO @@ -921,16 +923,6 @@ CONTAINS CALL close_file(unit_number=unit_number) CALL section_release(root_section) - ! References - CALL open_file(unit_number=unit_number, file_name="references.html", & - file_action="WRITE", file_status="REPLACE") - WRITE (unit_number, FMT='(A)') "The cp2k literature list" - WRITE (unit_number, FMT='(A)') "

CP2K references

" - CALL print_all_references(sorted=.TRUE., cited_only=.FALSE., & - FORMAT=print_format_html, unit=unit_number) - WRITE (unit_number, FMT='(A)') "" - CALL close_file(unit_number=unit_number) - ! Units CALL open_file(unit_number=unit_number, file_name="units.html", & file_action="WRITE", file_status="REPLACE") diff --git a/tools/docker/scripts/test_manual.sh b/tools/docker/scripts/test_manual.sh index e6dfe2fea3..18126a0131 100755 --- a/tools/docker/scripts/test_manual.sh +++ b/tools/docker/scripts/test_manual.sh @@ -48,7 +48,7 @@ cd /workspace/artifacts/manual set +e # disable error trapping for remainder of script ( set -e # abort if error is encountered - /opt/cp2k/docs/generate_input_reference.py ./cp2k_input.xml ./references.html + /opt/cp2k/docs/generate_input_reference.py ./cp2k_input.xml echo "" sphinx-build /opt/cp2k/docs/ /workspace/artifacts/manual -W -n --keep-going --jobs 16 /opt/cp2k/docs/fix_github_links.py /workspace/artifacts/manual