ENH: Support translated notebooks (#600)

* fall back to myst-parser for translated strings

* add internationalization test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* TST: use old sphinx generated comparison file

* TST: ignoring metadata missing from old sphinx generated files

* MAINT: cleanup debug prints

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: Brigitta Sipőcz <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

myst_nb/sphinx_.py

@@ -84,7 +84,7 @@ def parse(self, inputstring: str, document: nodes.document) -> None:
         # create a reader for the notebook
         nb_reader = create_nb_reader(document_path, md_config, nb_config, inputstring)
         # If the nb_reader is None, then we default to a standard Markdown parser
-        if nb_reader is None:
+        if nb_reader is None or document.current_source.endswith("<translated>"):
             return super().parse(inputstring, document)
         notebook = nb_reader.read(inputstring)
 

tests/conftest.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 import re
 import uuid
+import shutil
 
 import bs4
 from docutils.nodes import image as image_node
@@ -192,6 +193,8 @@ def sphinx_run(sphinx_params, make_app, tmp_path):
         )
         + "\n"
     )
+    if "language" in conf:
+        shutil.copytree(TEST_FILE_DIR / "locale", srcdir / "locale")
 
     for nb_file in sphinx_params["files"]:
         nb_path = TEST_FILE_DIR.joinpath(nb_file)
@@ -293,6 +296,9 @@ class FileRegression:
         re.escape(" translation_progress=\"{'total': 0, 'translated': 0}\""),
         # TODO: Remove when support for Sphinx<7.2 is dropped,
         r"original_uri=\"[^\"]*\"\s",
+        # TODO: Remove when support for Sphinx<8 is dropped,
+        re.escape(' translated="True"'),
+        re.escape(" translation_progress=\"{'total': 4, 'translated': 2}\""),
     )
 
     def __init__(self, file_regression):

tests/notebooks/basic_run_intl.ipynb

@@ -0,0 +1,53 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# a title\n",
+        "\n",
+        "some text\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {},
+      "source": [
+        "a=1\n",
+        "print(a)"
+      ],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "1\n"
+          ]
+        }
+      ]
+    }
+  ],
+  "metadata": {
+    "test_name": "notebook1",
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.1"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 2
+}

tests/notebooks/locale/es/LC_MESSAGES/basic_run_intl.po

@@ -0,0 +1,25 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C)
+# This file is distributed under the same license as the Project name not set package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: Project name not set \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2024-09-21 23:48+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <[email protected]>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: ../tests/notebooks/basic_run_intl.ipynb:10002
+msgid "a title"
+msgstr "un título"
+
+#: ../tests/notebooks/basic_run_intl.ipynb:10004
+msgid "some text"
+msgstr "algo de texto"

tests/test_parser.py

@@ -8,7 +8,6 @@
 @pytest.mark.sphinx_params("basic_run.ipynb", conf={"nb_execution_mode": "off"})
 def test_basic_run(sphinx_run, file_regression):
     sphinx_run.build()
-    # print(sphinx_run.status())
     assert sphinx_run.warnings() == ""
     assert set(sphinx_run.env.metadata["basic_run"].keys()) == {
         "test_name",
@@ -36,6 +35,38 @@ def test_basic_run(sphinx_run, file_regression):
     assert filenames == {"basic_run.ipynb"}
 
 
+@pytest.mark.sphinx_params(
+    "basic_run_intl.ipynb", conf={"language": "es", "locale_dirs": ["locale"]}
+)
+def test_basic_run_intl(sphinx_run, file_regression):
+    sphinx_run.build()
+    assert sphinx_run.warnings() == ""
+    assert set(sphinx_run.env.metadata["basic_run_intl"].keys()) == {
+        "test_name",
+        "wordcount",
+        "kernelspec",
+        "language_info",
+    }
+    assert set(sphinx_run.env.nb_metadata["basic_run_intl"].keys()) == set()
+    assert sphinx_run.env.metadata["basic_run_intl"]["test_name"] == "notebook1"
+    assert sphinx_run.env.metadata["basic_run_intl"]["kernelspec"] == {
+        "display_name": "Python 3",
+        "language": "python",
+        "name": "python3",
+    }
+    file_regression.check(
+        sphinx_run.get_doctree().pformat(), extension=".xml", encoding="utf-8"
+    )
+
+    filenames = {
+        p.name
+        for p in Path(
+            os.fspath(sphinx_run.app.srcdir / "_build" / "jupyter_execute")
+        ).iterdir()
+    }
+    assert filenames == {"basic_run_intl.ipynb"}
+
+
 @pytest.mark.sphinx_params("complex_outputs.ipynb", conf={"nb_execution_mode": "off"})
 def test_complex_outputs(sphinx_run, file_regression):
     sphinx_run.build()

tests/test_parser/test_basic_run_intl.xml

@@ -0,0 +1,14 @@
+<document source="basic_run_intl">
+    <section ids="a-title" names="a\ title un\ título">
+        <title>
+            un título
+        <paragraph>
+            algo de texto
+        <container cell_index="1" cell_metadata="{}" classes="cell" exec_count="1" nb_element="cell_code">
+            <container classes="cell_input" nb_element="cell_code_source">
+                <literal_block language="ipython3" xml:space="preserve">
+                    a=1
+                    print(a)
+            <container classes="cell_output" nb_element="cell_code_output">
+                <literal_block classes="output stream" language="myst-ansi" xml:space="preserve">
+                    1