diff --git a/.dockerignore b/.dockerignore index b7fb0934f6..30dbf96302 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,8 @@ obj regtesting tools/toolchain/install tools/toolchain/build +docs/_build +docs/CP2K_INPUT .git* *~ \#*\# diff --git a/.gitignore b/.gitignore index 083232e373..a98d530298 100644 --- a/.gitignore +++ b/.gitignore @@ -97,7 +97,6 @@ CMakeCache.txt CMakeFiles CMakeScripts Testing -Makefile cmake_install.cmake install_manifest.txt compile_commands.json @@ -253,6 +252,9 @@ instance/ # Sphinx documentation docs/_build/ +docs/CP2K_INPUT/ +docs/CP2K_INPUT.md +docs/bibliography.md # PyBuilder .pybuilder/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000..e359ddab95 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) --jobs 16 diff --git a/docs/_static/cp2k_manual_logo.png b/docs/_static/cp2k_manual_logo.png new file mode 120000 index 0000000000..3d06c1f62f --- /dev/null +++ b/docs/_static/cp2k_manual_logo.png @@ -0,0 +1 @@ +../../tools/manual/cp2k_manual_logo.png \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000000..b5a461b46c --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,41 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + + +# Disable benign warning: https://github.com/sphinx-doc/sphinx/blob/d3c91f951255c6729a53e38c895ddc0af036b5b9/sphinx/domains/python.py#L1283 +import logging + +logging.getLogger("sphinx.sphinx.domains.python").setLevel(logging.ERROR) + +# add_module_names = False + +project = "CP2K" +copyright = "2023, CP2K Developers" +author = "CP2K Developers" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["myst_parser", "sphinx_rtd_theme"] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +suppress_warnings = ["app", "ref", "index"] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] + +# add_module_names = False + +# https://myst-parser.readthedocs.io/en/v0.16.1/syntax/optional.html#syntax-header-anchors + +# EOF diff --git a/docs/generate_input_reference.py b/docs/generate_input_reference.py new file mode 100755 index 0000000000..d5bc8c4abc --- /dev/null +++ b/docs/generate_input_reference.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 + +# author: Ole Schuett + +from typing import Tuple, List, Optional +import lxml.etree as ET +import lxml +from pathlib import Path +import re +import sys + +SectionPath = Tuple[str, ...] + + +# ======================================================================================= +def main() -> None: + if len(sys.argv) != 3: + print("generate_input_reference.py ") + sys.exit(1) + + cp2k_input_xml_fn, references_html_fn = sys.argv[1:] + output_dir = Path(__file__).resolve().parent + + build_bibliography(references_html_fn, output_dir) + build_input_reference(cp2k_input_xml_fn, output_dir) + + +# ======================================================================================= +def build_bibliography(references_html_fn: str, output_dir: Path) -> None: + content = Path(references_html_fn).read_text() + entries = re.findall(".*?", content, re.DOTALL) + + output = [] + output += ["%", "% This file was created by generate_input_reference.py", "%"] + output += [f"# Bibliography", ""] + + for entry in entries: + pattern = '\[(.*?)\]\n (.*?)
(.*?)' + parts = re.search(pattern, entry, re.DOTALL) + assert parts + key = parts.group(1) + title = parts.group(3).strip() + if "
" in parts.group(2): + m = re.match("(.*?)
(.*)", parts.group(2), re.DOTALL) + assert m + authors, mix = m.groups() + else: + authors, mix = "", parts.group(2) + + if "https://doi.org" in mix: + m = re.match('\s*(.*?)', mix, re.DOTALL) + assert m + doi, ref = m.groups() + else: + doi, ref = "", mix.strip() + + output += [f"({key})=", f"## {key}", ""] + if doi: + output += [f"{authors} **{title}** _[{ref}]({doi})_", ""] + else: + output += [f"{authors} **{title}** _{ref}_", ""] + + # Write output + filename = output_dir / "bibliography.md" + filename.write_text("\n".join(output)) + print(f"Wrote {filename}") + + +# ======================================================================================= +def build_input_reference(cp2k_input_xml_fn: str, output_dir: Path) -> None: + tree = ET.parse(cp2k_input_xml_fn) + root = tree.getroot() + num_files_written = process_section(root, ("CP2K_INPUT",), output_dir) + + # Build landing page. + cp2k_version = get_text(root.find("CP2K_VERSION")) + compile_revision = get_text(root.find("COMPILE_REVISION")) + # cp2k_year = get_text(root.find("CP2K_YEAR")) + # compile_date = get_text(root.find("COMPILE_DATE")) + + output = [] + output += ["%", "% This file was created by generate_input_reference.py", "%"] + output += [f"# Input reference", ""] + + assert compile_revision.startswith("git:") + github_url = f"https://github.com/cp2k/cp2k/tree/{compile_revision[4:]}" + output += [f"Based on {cp2k_version} ([{compile_revision}]({github_url}))", ""] + + output += ["```{toctree}"] + output += [":maxdepth: 1"] + output += [":titlesonly:"] + output += [":caption: Top-level sections"] + output += [":glob:", ""] + output += ["CP2K_INPUT/*", ""] + + # Write output + filename = output_dir / "CP2K_INPUT.md" # Overwrite generic file. + filename.write_text("\n".join(output)) + print(f"Wrote markdown files for {num_files_written} input sections.") + + +# ======================================================================================= +def process_section( + section: lxml.etree._Element, section_path: SectionPath, output_dir: Path +) -> int: + # Find more section fields. + repeats = "repeats" in section.attrib and section.attrib["repeats"] == "yes" + description = get_text(section.find("DESCRIPTION")) + location = get_text(section.find("LOCATION")) + section_name = section_path[-1] # section.find("NAME") doesn't work for root + + # Find section references. + references = [get_text(ref.find("NAME")) for ref in section.findall("REFERENCE")] + + output = [] + output += ["%", "% This file was created by generate_input_reference.py", "%"] + output += [f"# {section_name}", ""] + if repeats: + output += [f"**Section can be repeated.**", ""] + if references: + citations = ", ".join([f"{{ref}}`{r}`" for r in references]) + output += [ + f"**References:** {citations}", + "", + ] + output += [f"{escape_markdown(description)} {github_link(location)}", ""] + + # Render TOC + if section.findall("SECTION"): + output += ["```{toctree}"] + output += [":maxdepth: 1"] + output += [":titlesonly:"] + output += [":caption: Subsections"] + output += [":glob:", ""] + output += [f"{section_name}/*"] # TODO maybe list subsection explicitly. + output += ["```", ""] + + # Render keywords + keywords = ( + section.findall("SECTION_PARAMETERS") + + section.findall("DEFAULT_KEYWORD") + + section.findall("KEYWORD") + ) + if keywords: + output += [f"## Keywords", ""] + for keyword in keywords: + output += render_keyword(keyword, section_path) + + # Write output + section_dir = output_dir / "/".join(section_path[:-1]) + section_dir.mkdir(exist_ok=True) + filename = section_dir / f"{section_name}.md" + filename.write_text("\n".join(output)) + num_files_written = 1 + + # Process subsections + for subsection in section.findall("SECTION"): + subsection_name_element = subsection.find("NAME") + subsection_name = get_text(subsection.find("NAME")) + subsection_path = (*section_path, subsection_name) + num_files_written += process_section(subsection, subsection_path, output_dir) + + return num_files_written + + +# ======================================================================================= +def render_keyword( + keyword: lxml.etree._Element, section_path: SectionPath +) -> List[str]: + # Find keyword names. + keyword_names: List[str] + if keyword.tag == "SECTION_PARAMETERS": + keyword_names = ["SECTION_PARAMETERS"] + elif keyword.tag == "DEFAULT_KEYWORD": + keyword_names = ["DEFAULT_KEYWORD"] + else: + keyword_names = [get_text(name) for name in keyword.findall("NAME")] + assert keyword_names + + # Find more keyword fields. + default_value = get_text(keyword.find("DEFAULT_VALUE")) + usage = get_text(keyword.find("USAGE")) + description = get_text(keyword.find("DESCRIPTION")) + location = get_text(keyword.find("LOCATION")) + lone_leyword_value = get_text(keyword.find("LONE_KEYWORD_VALUE")) + + # Find keyword data type. + data_type_element = keyword.find("DATA_TYPE") + assert data_type_element is not None + data_type = data_type_element.attrib["kind"] + if data_type == "word": + data_type = "string" + if data_type == "keyword": + data_type = "enum" + + # Need to distiguish between multiple values (n_var) and repeating keyword. + repeats = keyword.attrib["repeats"] == "yes" + n_var = int(get_text(data_type_element.find("N_VAR"))) + + # Find keyword references. + references = [get_text(ref.find("NAME")) for ref in keyword.findall("REFERENCE")] + + # Skip removed keywords. + if keyword.attrib.get("removed", "no") == "yes": + print(f"Skipping removed keyword: {keyword_names[0]}") + return [] + + # To get references to work we'd have to encode the `section_path` as `:module:`. + # We could then also set `add_module_names = False` in the config and re-enable + # the warnings for the sphinx.domains.python module. + # However, the links would not be backwards compatible. A solution might be + # a combinations of explicit targets and myst_heading_slug_func in the config. + output: List[str] = [] + output += [f"```{{py:data}} {keyword_names[0]}"] + n_var_brackets = f"[{n_var}]" if n_var > 1 else "" + output += [f":type: {data_type}{n_var_brackets}"] + if default_value: + output += [f":value: {default_value}"] + output += [""] + if len(keyword_names) > 1: + aliases = " ,".join(keyword_names) + output += [f"**Aliase:** {aliases}"] + if repeats: + output += [f"**Keyword can be repeated.**", ""] + if lone_leyword_value: + output += [f"**Lone keyword:** `{escape_markdown(lone_leyword_value)}`", ""] + if usage: + output += [ + f"**Usage:** _{escape_markdown(usage)}_", + "", + ] + if data_type == "enum": + output += [f"**Valid values:**"] + for item in keyword.findall("DATA_TYPE/ENUMERATION/ITEM"): + item_name = get_text(item.find("NAME")) + item_description = get_text(item.find("DESCRIPTION")) + output += [f"* `{item_name}` {escape_markdown(item_description)}"] + output += [""] + if references: + citations = ", ".join([f"{{ref}}`{r}`" for r in references]) + output += [ + f"**References:** {citations}", + "", + ] + output += [f"{escape_markdown(description)} {github_link(location)}", ""] + + output += ["```", ""] # Close py:data directive. + + return output + + +# ======================================================================================= +def get_text(element: Optional[lxml.etree._Element]) -> str: + if element is not None: + if element.text is not None: + return element.text + return "" + + +# ======================================================================================= +def escape_markdown(text: str) -> str: + text = text.replace("__", "\_\_") + return text + + +# ======================================================================================= +def github_link(location: str) -> str: + if not location: + return "" + location_url = location.replace(":", "#L") + github_url = f"https://github.com/cp2k/cp2k/blob/master/src/{location_url}" + return f"[[Edit on GitHub]({github_url})]" + + +# ======================================================================================= + +main() + +# EOF diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000000..be124c8e7e --- /dev/null +++ b/docs/index.md @@ -0,0 +1,29 @@ +# Welcome to the CP2K Manual ! + +```{image} _static/cp2k_manual_logo.png +:width: 250px +:align: center +:class: no-scaled-link +``` + +% TODO: `{toctree} % :caption: Get started % % get-started/installation % get-started/tutorials % ` + +% TODO: `{toctree} % :caption: How-to guides % % dummy % ` + +```{toctree} +:caption: References +:titlesonly: +:maxdepth: 1 + +CP2K_INPUT +bibliography +units +``` + +% TODO: `{toctree} % :caption: Explanations % % dummy % ` + +# Indices and tables + +- {ref}`genindex` +- {ref}`modindex` +- {ref}`search` diff --git a/docs/units.md b/docs/units.md new file mode 100644 index 0000000000..dfc4546da7 --- /dev/null +++ b/docs/units.md @@ -0,0 +1,97 @@ +# Units + +CP2K Available Units of Measurement + +## Energy + +Possible units of measurement for Energies. The `[energy]` entry acts like a dummy flag (assumes the unit of measurement of energy is in internal units), useful for dimensional analysis. + +- `hartree` +- `wavenumber_e` +- `joule` +- `kcalmol` +- `kjmol` +- `Ry` +- `eV` +- `keV` +- `K_e` +- `energy` + +## Length + +Possible units of measurement for Lengths. The `[length]` entry acts like a dummy flag (assumes the unit of measurement of length is in internal units), useful for dimensional analysis. + +- `bohr` +- `m` +- `pm` +- `nm` +- `angstrom` +- `length` + +## Temperature + +Possible units of measurement for Temperature. The `[temperature]` entry acts like a dummy flag (assumes the unit of measurement of temperature is in internal units), useful for dimensional analysis. + +- `K` +- `au_temp` +- `temperature` + +## Pressure + +Possible units of measurement for Pressure. The `[pressure]` entry acts like a dummy flag (assumes the unit of measurement of pressure is in internal units), useful for dimensional analysis. + +- `bar` +- `atm` +- `kbar` +- `Pa` +- `MPa` +- `GPa` +- `au_p` +- `pressure` + +## Angle + +Possible units of measurement for Angles. The `[angle]` entry acts like a dummy flag (assumes the unit of measurement of angle is in internal units), useful for dimensional analysis. + +- `rad` +- `deg` +- `angle` + +## Time + +Possible units of measurement for Time. The `[time]` entry acts like a dummy flag (assumes the unit of measurement of time is in internal units), useful for dimensional analysis. + +- `s` +- `fs` +- `ps` +- `au_t` +- `wavenumber_t` +- `time` + +## Mass + +Possible units of measurement for Masses. The `[mass`\] entry acts like a dummy flag (assumes the unit of measurement of mass is in internal units), useful for dimensional analysis. + +- `kg` +- `amu` +- `m_e` +- `mass` + +## Potential + +Possible units of measurement for potentials. The `[potential]` entry acts like a dummy flag (assumes the unit of measurement of potential is in internal units), useful for dimensional analysis. + +- `volt` +- `au_pot` +- `potential` + +## Force + +Possible units of measurement for forces. The `[force]` entry acts like a dummy flag (assumes the unit of measurement of force is in internal units), useful for dimensional analysis. + +- `N` +- `Newton` +- `mN` +- `mNewton` +- `au_f` +- `force` diff --git a/tools/docker/Dockerfile.test_doxygen b/tools/docker/Dockerfile.test_doxygen index 42b18bdb3f..c0ad296c63 100644 --- a/tools/docker/Dockerfile.test_doxygen +++ b/tools/docker/Dockerfile.test_doxygen @@ -15,6 +15,7 @@ ARG GIT_COMMIT_SHA COPY ./src ./src COPY ./exts ./exts COPY ./data ./data +COPY ./docs ./docs COPY ./tools ./tools COPY ./cmake ./cmake COPY ./CMakeLists.txt . diff --git a/tools/docker/Dockerfile.test_manual b/tools/docker/Dockerfile.test_manual index ab165d8dd9..fb5fb29057 100644 --- a/tools/docker/Dockerfile.test_manual +++ b/tools/docker/Dockerfile.test_manual @@ -82,6 +82,7 @@ COPY ./tools/regtesting ./tools/regtesting # Generate manual. COPY ./tools/manual ./tools/manual COPY ./tools/input_editing ./tools/input_editing +COPY ./docs ./docs COPY ./tools/docker/scripts/test_manual.sh . ARG ADD_EDIT_LINKS=yes RUN ./test_manual.sh "${ADD_EDIT_LINKS}" 2>&1 | tee report.log diff --git a/tools/docker/Dockerfile.test_misc b/tools/docker/Dockerfile.test_misc index cd6046e59c..9854fb5ea1 100644 --- a/tools/docker/Dockerfile.test_misc +++ b/tools/docker/Dockerfile.test_misc @@ -15,6 +15,7 @@ ARG GIT_COMMIT_SHA COPY ./src ./src COPY ./exts ./exts COPY ./data ./data +COPY ./docs ./docs COPY ./tools ./tools COPY ./cmake ./cmake COPY ./CMakeLists.txt . diff --git a/tools/docker/generate_dockerfiles.py b/tools/docker/generate_dockerfiles.py index d77d8ba06a..1829c6e339 100755 --- a/tools/docker/generate_dockerfiles.py +++ b/tools/docker/generate_dockerfiles.py @@ -228,6 +228,7 @@ def manual() -> str: # Generate manual. COPY ./tools/manual ./tools/manual COPY ./tools/input_editing ./tools/input_editing +COPY ./docs ./docs COPY ./tools/docker/scripts/test_manual.sh . ARG ADD_EDIT_LINKS=yes RUN ./test_manual.sh "${{ADD_EDIT_LINKS}}" 2>&1 | tee report.log @@ -287,6 +288,7 @@ ARG GIT_COMMIT_SHA COPY ./src ./src COPY ./exts ./exts COPY ./data ./data +COPY ./docs ./docs COPY ./tools ./tools COPY ./cmake ./cmake COPY ./CMakeLists.txt . diff --git a/tools/docker/scripts/install_misc.sh b/tools/docker/scripts/install_misc.sh index 25ad42ef2a..e10129b14f 100755 --- a/tools/docker/scripts/install_misc.sh +++ b/tools/docker/scripts/install_misc.sh @@ -25,8 +25,9 @@ pip3 install --quiet \ numpy \ matplotlib \ requests \ + types-lxml \ types-requests \ - mypy==0.991 + mypy==1.3.0 # download inputs for minimax_to_fortran_source.py wget -q https://www.cp2k.org/static/downloads/1_xData.zip diff --git a/tools/docker/scripts/test_manual.sh b/tools/docker/scripts/test_manual.sh index 82c5649d3f..179f8c49d4 100755 --- a/tools/docker/scripts/test_manual.sh +++ b/tools/docker/scripts/test_manual.sh @@ -29,9 +29,13 @@ echo -e "\n========== Installing Dependencies ==========" apt-get update -qq apt-get install -qq --no-install-recommends \ default-jre-headless \ - libsaxonhe-java + libsaxonhe-java \ + python3 \ + python3-pip rm -rf /var/lib/apt/lists/* +pip3 install --quiet sphinx myst-parser sphinx_rtd_theme lxml + echo -e "\n========== Generating Manual ==========" mkdir -p /workspace/artifacts/manual @@ -53,13 +57,27 @@ set +e # disable error trapping for remainder of script $SAXON -o:cp2k.vim ./cp2k_input.xml ${TOOLS}/input_editing/vim/vim.xsl ) EXIT_CODE=$? - if ((EXIT_CODE)); then - echo -e "\nSummary: Something is wrong." + echo -e "\nSummary: Saxon run failed." echo -e "Status: FAILED\n" -else - echo -e "\nSummary: Manual generation works fine." - echo -e "Status: OK\n" + exit fi +echo -e "\n========== Generating New Manual ==========" +( + set -e # abort if error is encountered + /opt/cp2k/docs/generate_input_reference.py ./cp2k_input.xml ./references.html + echo "" + sphinx-build /opt/cp2k/docs/ /workspace/artifacts/manual/new --jobs 16 +) +EXIT_CODE=$? +if ((EXIT_CODE)); then + echo -e "\nSummary: Sphinx build failed" + echo -e "Status: FAILED\n" + exit +fi + +echo -e "\nSummary: Manual generation works fine." +echo -e "Status: OK\n" + #EOF diff --git a/tools/docker/scripts/test_misc.sh b/tools/docker/scripts/test_misc.sh index 33ae191b4e..96fe317cf4 100755 --- a/tools/docker/scripts/test_misc.sh +++ b/tools/docker/scripts/test_misc.sh @@ -41,6 +41,7 @@ run_test mypy --strict ./tools/precommit/format_makefile.py run_test mypy --strict ./tools/docker/generate_dockerfiles.py run_test mypy --strict ./tools/apptainer/generate_apptainer_def_files.py run_test mypy --strict ./tools/conventions/analyze_gfortran_ast.py +run_test mypy --strict ./docs/generate_input_reference.py # TODO: Find a way to test generate_dashboard.py without git repository. # diff --git a/tools/precommit/precommit.py b/tools/precommit/precommit.py index 0f927b7574..d1abde2c67 100755 --- a/tools/precommit/precommit.py +++ b/tools/precommit/precommit.py @@ -98,6 +98,10 @@ def main() -> None: continue if root.startswith("./doxygen"): continue + if root.startswith("./docs/_build"): + continue + if root.startswith("./docs/CP2K_INPUT"): + continue if root.startswith("./exts"): continue if root.startswith("./obj"):