Docs: Add Sphinx-based manual

.dockerignore

@@ -4,6 +4,8 @@ obj
 regtesting
 tools/toolchain/install
 tools/toolchain/build
+docs/_build
+docs/CP2K_INPUT
 .git*
 *~
 \#*\#

.gitignore

@@ -97,7 +97,6 @@ CMakeCache.txt
 CMakeFiles
 CMakeScripts
 Testing
-Makefile
 cmake_install.cmake
 install_manifest.txt
 compile_commands.json
@@ -253,6 +252,9 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/CP2K_INPUT/
+docs/CP2K_INPUT.md
+docs/bibliography.md
 
 # PyBuilder
 .pybuilder/

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) --jobs 16

docs/_static/cp2k_manual_logo.png

@@ -0,0 +1 @@
+../../tools/manual/cp2k_manual_logo.png

docs/conf.py

@@ -0,0 +1,41 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+
+# Disable benign warning: https://github.com/sphinx-doc/sphinx/blob/d3c91f951255c6729a53e38c895ddc0af036b5b9/sphinx/domains/python.py#L1283
+import logging
+
+logging.getLogger("sphinx.sphinx.domains.python").setLevel(logging.ERROR)
+
+# add_module_names = False
+
+project = "CP2K"
+copyright = "2023, CP2K Developers"
+author = "CP2K Developers"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ["myst_parser", "sphinx_rtd_theme"]
+
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+suppress_warnings = ["app", "ref", "index"]
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "sphinx_rtd_theme"
+html_static_path = ["_static"]
+
+# add_module_names = False
+
+# https://myst-parser.readthedocs.io/en/v0.16.1/syntax/optional.html#syntax-header-anchors
+
+# EOF

docs/generate_input_reference.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+
+# author: Ole Schuett
+
+from typing import Tuple, List, Optional
+import lxml.etree as ET
+import lxml
+from pathlib import Path
+import re
+import sys
+
+SectionPath = Tuple[str, ...]
+
+
+# =======================================================================================
+def main() -> None:
+    if len(sys.argv) != 3:
+        print("generate_input_reference.py <cp2k_input.xml> <references.html>")
+        sys.exit(1)
+
+    cp2k_input_xml_fn, references_html_fn = sys.argv[1:]
+    output_dir = Path(__file__).resolve().parent
+
+    build_bibliography(references_html_fn, output_dir)
+    build_input_reference(cp2k_input_xml_fn, output_dir)
+
+
+# =======================================================================================
+def build_bibliography(references_html_fn: str, output_dir: Path) -> None:
+    content = Path(references_html_fn).read_text()
+    entries = re.findall("<TR>.*?</TR>", content, re.DOTALL)
+
+    output = []
+    output += ["%", "% This file was created by generate_input_reference.py", "%"]
+    output += [f"# Bibliography", ""]
+
+    for entry in entries:
+        pattern = '<TR><TD>\[(.*?)\]</TD><TD>\n <A NAME="reference_\d+">(.*?)</A><br>(.*?)</TD></TR>'
+        parts = re.search(pattern, entry, re.DOTALL)
+        assert parts
+        key = parts.group(1)
+        title = parts.group(3).strip()
+        if "<br>" in parts.group(2):
+            m = re.match("(.*?)<br>(.*)", parts.group(2), re.DOTALL)
+            assert m
+            authors, mix = m.groups()
+        else:
+            authors, mix = "", parts.group(2)
+
+        if "https://doi.org" in mix:
+            m = re.match('\s*<A HREF="(.*?)">(.*?)</A>', mix, re.DOTALL)
+            assert m
+            doi, ref = m.groups()
+        else:
+            doi, ref = "", mix.strip()
+
+        output += [f"({key})=", f"## {key}", ""]
+        if doi:
+            output += [f"{authors} **{title}** _[{ref}]({doi})_", ""]
+        else:
+            output += [f"{authors} **{title}** _{ref}_", ""]
+
+    # Write output
+    filename = output_dir / "bibliography.md"
+    filename.write_text("\n".join(output))
+    print(f"Wrote {filename}")
+
+
+# =======================================================================================
+def build_input_reference(cp2k_input_xml_fn: str, output_dir: Path) -> None:
+    tree = ET.parse(cp2k_input_xml_fn)
+    root = tree.getroot()
+    num_files_written = process_section(root, ("CP2K_INPUT",), output_dir)
+
+    # Build landing page.
+    cp2k_version = get_text(root.find("CP2K_VERSION"))
+    compile_revision = get_text(root.find("COMPILE_REVISION"))
+    # cp2k_year = get_text(root.find("CP2K_YEAR"))
+    # compile_date = get_text(root.find("COMPILE_DATE"))
+
+    output = []
+    output += ["%", "% This file was created by generate_input_reference.py", "%"]
+    output += [f"# Input reference", ""]
+
+    assert compile_revision.startswith("git:")
+    github_url = f"https://github.com/cp2k/cp2k/tree/{compile_revision[4:]}"
+    output += [f"Based on {cp2k_version} ([{compile_revision}]({github_url}))", ""]
+
+    output += ["```{toctree}"]
+    output += [":maxdepth: 1"]
+    output += [":titlesonly:"]
+    output += [":caption: Top-level sections"]
+    output += [":glob:", ""]
+    output += ["CP2K_INPUT/*", ""]
+
+    # Write output
+    filename = output_dir / "CP2K_INPUT.md"  # Overwrite generic file.
+    filename.write_text("\n".join(output))
+    print(f"Wrote markdown files for {num_files_written} input sections.")
+
+
+# =======================================================================================
+def process_section(
+    section: lxml.etree._Element, section_path: SectionPath, output_dir: Path
+) -> int:
+    # Find more section fields.
+    repeats = "repeats" in section.attrib and section.attrib["repeats"] == "yes"
+    description = get_text(section.find("DESCRIPTION"))
+    location = get_text(section.find("LOCATION"))
+    section_name = section_path[-1]  # section.find("NAME") doesn't work for root
+
+    # Find section references.
+    references = [get_text(ref.find("NAME")) for ref in section.findall("REFERENCE")]
+
+    output = []
+    output += ["%", "% This file was created by generate_input_reference.py", "%"]
+    output += [f"# {section_name}", ""]
+    if repeats:
+        output += [f"**Section can be repeated.**", ""]
+    if references:
+        citations = ", ".join([f"{{ref}}`{r}`" for r in references])
+        output += [
+            f"**References:** {citations}",
+            "",
+        ]
+    output += [f"{escape_markdown(description)} {github_link(location)}", ""]
+
+    # Render TOC
+    if section.findall("SECTION"):
+        output += ["```{toctree}"]
+        output += [":maxdepth: 1"]
+        output += [":titlesonly:"]
+        output += [":caption: Subsections"]
+        output += [":glob:", ""]
+        output += [f"{section_name}/*"]  # TODO maybe list subsection explicitly.
+        output += ["```", ""]
+
+    # Render keywords
+    keywords = (
+        section.findall("SECTION_PARAMETERS")
+        + section.findall("DEFAULT_KEYWORD")
+        + section.findall("KEYWORD")
+    )
+    if keywords:
+        output += [f"## Keywords", ""]
+        for keyword in keywords:
+            output += render_keyword(keyword, section_path)
+
+    # Write output
+    section_dir = output_dir / "/".join(section_path[:-1])
+    section_dir.mkdir(exist_ok=True)
+    filename = section_dir / f"{section_name}.md"
+    filename.write_text("\n".join(output))
+    num_files_written = 1
+
+    # Process subsections
+    for subsection in section.findall("SECTION"):
+        subsection_name_element = subsection.find("NAME")
+        subsection_name = get_text(subsection.find("NAME"))
+        subsection_path = (*section_path, subsection_name)
+        num_files_written += process_section(subsection, subsection_path, output_dir)
+
+    return num_files_written
+
+
+# =======================================================================================
+def render_keyword(
+    keyword: lxml.etree._Element, section_path: SectionPath
+) -> List[str]:
+    # Find keyword names.
+    keyword_names: List[str]
+    if keyword.tag == "SECTION_PARAMETERS":
+        keyword_names = ["SECTION_PARAMETERS"]
+    elif keyword.tag == "DEFAULT_KEYWORD":
+        keyword_names = ["DEFAULT_KEYWORD"]
+    else:
+        keyword_names = [get_text(name) for name in keyword.findall("NAME")]
+    assert keyword_names
+
+    # Find more keyword fields.
+    default_value = get_text(keyword.find("DEFAULT_VALUE"))
+    usage = get_text(keyword.find("USAGE"))
+    description = get_text(keyword.find("DESCRIPTION"))
+    location = get_text(keyword.find("LOCATION"))
+    lone_leyword_value = get_text(keyword.find("LONE_KEYWORD_VALUE"))
+
+    # Find keyword data type.
+    data_type_element = keyword.find("DATA_TYPE")
+    assert data_type_element is not None
+    data_type = data_type_element.attrib["kind"]
+    if data_type == "word":
+        data_type = "string"
+    if data_type == "keyword":
+        data_type = "enum"
+
+    # Need to distiguish between multiple values (n_var) and repeating keyword.
+    repeats = keyword.attrib["repeats"] == "yes"
+    n_var = int(get_text(data_type_element.find("N_VAR")))
+
+    # Find keyword references.
+    references = [get_text(ref.find("NAME")) for ref in keyword.findall("REFERENCE")]
+
+    # Skip removed keywords.
+    if keyword.attrib.get("removed", "no") == "yes":
+        print(f"Skipping removed keyword: {keyword_names[0]}")
+        return []
+
+    # To get references to work we'd have to encode the `section_path` as `:module:`.
+    # We could then also set `add_module_names = False` in the config and re-enable
+    # the warnings for the sphinx.domains.python module.
+    # However, the links would not be backwards compatible. A solution might be
+    # a combinations of explicit targets and myst_heading_slug_func in the config.
+    output: List[str] = []
+    output += [f"```{{py:data}}  {keyword_names[0]}"]
+    n_var_brackets = f"[{n_var}]" if n_var > 1 else ""
+    output += [f":type: {data_type}{n_var_brackets}"]
+    if default_value:
+        output += [f":value: {default_value}"]
+    output += [""]
+    if len(keyword_names) > 1:
+        aliases = " ,".join(keyword_names)
+        output += [f"**Aliase:** {aliases}"]
+    if repeats:
+        output += [f"**Keyword can be repeated.**", ""]
+    if lone_leyword_value:
+        output += [f"**Lone keyword:** `{escape_markdown(lone_leyword_value)}`", ""]
+    if usage:
+        output += [
+            f"**Usage:** _{escape_markdown(usage)}_",
+            "",
+        ]
+    if data_type == "enum":
+        output += [f"**Valid values:**"]
+        for item in keyword.findall("DATA_TYPE/ENUMERATION/ITEM"):
+            item_name = get_text(item.find("NAME"))
+            item_description = get_text(item.find("DESCRIPTION"))
+            output += [f"* `{item_name}` {escape_markdown(item_description)}"]
+        output += [""]
+    if references:
+        citations = ", ".join([f"{{ref}}`{r}`" for r in references])
+        output += [
+            f"**References:** {citations}",
+            "",
+        ]
+    output += [f"{escape_markdown(description)} {github_link(location)}", ""]
+
+    output += ["```", ""]  # Close py:data directive.
+
+    return output
+
+
+# =======================================================================================
+def get_text(element: Optional[lxml.etree._Element]) -> str:
+    if element is not None:
+        if element.text is not None:
+            return element.text
+    return ""
+
+
+# =======================================================================================
+def escape_markdown(text: str) -> str:
+    text = text.replace("__", "\_\_")
+    return text
+
+
+# =======================================================================================
+def github_link(location: str) -> str:
+    if not location:
+        return ""
+    location_url = location.replace(":", "#L")
+    github_url = f"https://github.com/cp2k/cp2k/blob/master/src/{location_url}"
+    return f"<small>[[Edit on GitHub]({github_url})]</small>"
+
+
+# =======================================================================================
+
+main()
+
+# EOF

docs/index.md

@@ -0,0 +1,29 @@
+# Welcome to the CP2K Manual !
+
+```{image} _static/cp2k_manual_logo.png
+:width: 250px
+:align: center
+:class: no-scaled-link
+```
+
+% TODO: `{toctree} % :caption: Get started % % get-started/installation % get-started/tutorials % `
+
+% TODO: `{toctree} % :caption: How-to guides % % dummy % `
+
+```{toctree}
+:caption: References
+:titlesonly:
+:maxdepth: 1
+
+CP2K_INPUT
+bibliography
+units
+```
+
+% TODO: `{toctree} % :caption: Explanations % % dummy % `
+
+# Indices and tables
+
+- {ref}`genindex`
+- {ref}`modindex`
+- {ref}`search`

docs/units.md

@@ -0,0 +1,97 @@
+# Units
+
+CP2K Available Units of Measurement
+
+## Energy
+
+Possible units of measurement for Energies. The `[energy]` entry acts like a dummy flag (assumes the unit of measurement of energy is in internal units), useful for dimensional analysis.
+
+- `hartree`
+- `wavenumber_e`
+- `joule`
+- `kcalmol`
+- `kjmol`
+- `Ry`
+- `eV`
+- `keV`
+- `K_e`
+- `energy`
+
+## Length
+
+Possible units of measurement for Lengths. The `[length]` entry acts like a dummy flag (assumes the unit of measurement of length is in internal units), useful for dimensional analysis.
+
+- `bohr`
+- `m`
+- `pm`
+- `nm`
+- `angstrom`
+- `length`
+
+## Temperature
+
+Possible units of measurement for Temperature. The `[temperature]` entry acts like a dummy flag (assumes the unit of measurement of temperature is in internal units), useful for dimensional analysis.
+
+- `K`
+- `au_temp`
+- `temperature`
+
+## Pressure
+
+Possible units of measurement for Pressure. The `[pressure]` entry acts like a dummy flag (assumes the unit of measurement of pressure is in internal units), useful for dimensional analysis.
+
+- `bar`
+- `atm`
+- `kbar`
+- `Pa`
+- `MPa`
+- `GPa`
+- `au_p`
+- `pressure`
+
+## Angle
+
+Possible units of measurement for Angles. The `[angle]` entry acts like a dummy flag (assumes the unit of measurement of angle is in internal units), useful for dimensional analysis.
+
+- `rad`
+- `deg`
+- `angle`
+
+## Time
+
+Possible units of measurement for Time. The `[time]` entry acts like a dummy flag (assumes the unit of measurement of time is in internal units), useful for dimensional analysis.
+
+- `s`
+- `fs`
+- `ps`
+- `au_t`
+- `wavenumber_t`
+- `time`
+
+## Mass
+
+Possible units of measurement for Masses. The `[mass`\] entry acts like a dummy flag (assumes the unit of measurement of mass is in internal units), useful for dimensional analysis.
+
+- `kg`
+- `amu`
+- `m_e`
+- `mass`
+
+## Potential
+
+Possible units of measurement for potentials. The `[potential]` entry acts like a dummy flag (assumes the unit of measurement of potential is in internal units), useful for dimensional analysis.
+
+- `volt`
+- `au_pot`
+- `potential`
+
+## Force
+
+Possible units of measurement for forces. The `[force]` entry acts like a dummy flag (assumes the unit of measurement of force is in internal units), useful for dimensional analysis.
+
+- `N`
+- `Newton`
+- `mN`
+- `mNewton`
+- `au_f`
+- `force`

tools/docker/Dockerfile.test_doxygen

@@ -15,6 +15,7 @@ ARG GIT_COMMIT_SHA
 COPY ./src ./src
 COPY ./exts ./exts
 COPY ./data ./data
+COPY ./docs ./docs
 COPY ./tools ./tools
 COPY ./cmake ./cmake
 COPY ./CMakeLists.txt .

tools/docker/Dockerfile.test_manual

@@ -82,6 +82,7 @@ COPY ./tools/regtesting ./tools/regtesting
 # Generate manual.
 COPY ./tools/manual ./tools/manual
 COPY ./tools/input_editing ./tools/input_editing
+COPY ./docs ./docs
 COPY ./tools/docker/scripts/test_manual.sh .
 ARG ADD_EDIT_LINKS=yes
 RUN ./test_manual.sh "${ADD_EDIT_LINKS}" 2>&1 | tee report.log

tools/docker/Dockerfile.test_misc

@@ -15,6 +15,7 @@ ARG GIT_COMMIT_SHA
 COPY ./src ./src
 COPY ./exts ./exts
 COPY ./data ./data
+COPY ./docs ./docs
 COPY ./tools ./tools
 COPY ./cmake ./cmake
 COPY ./CMakeLists.txt .

tools/docker/generate_dockerfiles.py

@@ -228,6 +228,7 @@ def manual() -> str:
 # Generate manual.
 COPY ./tools/manual ./tools/manual
 COPY ./tools/input_editing ./tools/input_editing
+COPY ./docs ./docs
 COPY ./tools/docker/scripts/test_manual.sh .
 ARG ADD_EDIT_LINKS=yes
 RUN ./test_manual.sh "${{ADD_EDIT_LINKS}}" 2>&1 | tee report.log
@@ -287,6 +288,7 @@ ARG GIT_COMMIT_SHA
 COPY ./src ./src
 COPY ./exts ./exts
 COPY ./data ./data
+COPY ./docs ./docs
 COPY ./tools ./tools
 COPY ./cmake ./cmake
 COPY ./CMakeLists.txt .

tools/docker/scripts/install_misc.sh

@@ -25,8 +25,9 @@ pip3 install --quiet \
   numpy \
   matplotlib \
   requests \
+  types-lxml \
   types-requests \
-  mypy==0.991
+  mypy==1.3.0
 
 # download inputs for minimax_to_fortran_source.py
 wget -q https://www.cp2k.org/static/downloads/1_xData.zip

tools/docker/scripts/test_manual.sh

@@ -29,9 +29,13 @@ echo -e "\n========== Installing Dependencies =========="
 apt-get update -qq
 apt-get install -qq --no-install-recommends \
   default-jre-headless \
-  libsaxonhe-java
+  libsaxonhe-java \
+  python3 \
+  python3-pip
 rm -rf /var/lib/apt/lists/*
 
+pip3 install --quiet sphinx myst-parser sphinx_rtd_theme lxml
+
 echo -e "\n========== Generating Manual =========="
 
 mkdir -p /workspace/artifacts/manual
@@ -53,13 +57,27 @@ set +e # disable error trapping for remainder of script
   $SAXON -o:cp2k.vim ./cp2k_input.xml ${TOOLS}/input_editing/vim/vim.xsl
 )
 EXIT_CODE=$?
-
 if ((EXIT_CODE)); then
-  echo -e "\nSummary: Something is wrong."
+  echo -e "\nSummary: Saxon run failed."
   echo -e "Status: FAILED\n"
-else
-  echo -e "\nSummary: Manual generation works fine."
-  echo -e "Status: OK\n"
+  exit
 fi
 
+echo -e "\n========== Generating New Manual =========="
+(
+  set -e # abort if error is encountered
+  /opt/cp2k/docs/generate_input_reference.py ./cp2k_input.xml ./references.html
+  echo ""
+  sphinx-build /opt/cp2k/docs/ /workspace/artifacts/manual/new --jobs 16
+)
+EXIT_CODE=$?
+if ((EXIT_CODE)); then
+  echo -e "\nSummary: Sphinx build failed"
+  echo -e "Status: FAILED\n"
+  exit
+fi
+
+echo -e "\nSummary: Manual generation works fine."
+echo -e "Status: OK\n"
+
 #EOF

tools/docker/scripts/test_misc.sh

@@ -41,6 +41,7 @@ run_test mypy --strict ./tools/precommit/format_makefile.py
 run_test mypy --strict ./tools/docker/generate_dockerfiles.py
 run_test mypy --strict ./tools/apptainer/generate_apptainer_def_files.py
 run_test mypy --strict ./tools/conventions/analyze_gfortran_ast.py
+run_test mypy --strict ./docs/generate_input_reference.py
 
 # TODO: Find a way to test generate_dashboard.py without git repository.
 #

tools/precommit/precommit.py

@@ -98,6 +98,10 @@ def main() -> None:
                 continue
             if root.startswith("./doxygen"):
                 continue
+            if root.startswith("./docs/_build"):
+                continue
+            if root.startswith("./docs/CP2K_INPUT"):
+                continue
             if root.startswith("./exts"):
                 continue
             if root.startswith("./obj"):