👌 IMPROVE: [code load] Enable utf-8 as the standard encoding for reading code from files

docs/use/markdown.md

@@ -116,7 +116,8 @@ you specify a `load` metadata attribute such as:
 ````
 
 ```{warning}
-This is an experimental feature that is **not** part of the core `MyST` markup specification, and may be removed in the future. Using `:load:` will also overwrite any code written into the directive.
+This is an experimental feature that is **not** part of the core `MyST` markup specification, and may be removed in the future. Using `:load:` will also overwrite any code written into the directive. The
+default encoding is set to `utf-8` when reading files for all platforms.
 ```
 
 ### Syntax for markdown

myst_nb/converter.py

@@ -187,7 +187,7 @@ def read_cell_metadata(token, cell_index):
     return metadata
 
 
-def load_code_from_file(nb_path, file_name, token, body_lines):
+def load_code_from_file(nb_path, file_name, token, body_lines, encoding="utf-8"):
     """load source code from a file."""
     if nb_path is None:
         raise LoadFileParsingError("path to notebook not supplied for :load:")
@@ -200,7 +200,7 @@ def load_code_from_file(nb_path, file_name, token, body_lines):
         )
         LOGGER.warning(msg)
     try:
-        body_lines = file_path.read_text().split("\n")
+        body_lines = file_path.read_text(encoding=encoding).split("\n")
     except Exception:
         raise LoadFileParsingError("Can't read file from :load: {}".format(file_path))
     return body_lines

tests/notebooks/mystnb_codecell_file.py

@@ -1,3 +1,10 @@
 # flake8: noqa
 
 import numpy as np
+
+# Test: Unicode
+
+
+def c_star(x, β, γ):
+
+    return (1 - β ** (1 / γ)) * x

tests/test_mystnb_features/test_codecell_file.ipynb

@@ -2,6 +2,7 @@
  "cells": [
   {
    "cell_type": "markdown",
+   "id": "e67c115d",
    "metadata": {},
    "source": [
     "# a title"
@@ -10,14 +11,22 @@
   {
    "cell_type": "code",
    "execution_count": 1,
+   "id": "18613988",
    "metadata": {
-    "file": "mystnb_codecell_file.py"
+    "load": "mystnb_codecell_file.py"
    },
    "outputs": [],
    "source": [
     "# flake8: noqa\n",
     "\n",
-    "import numpy as np\n"
+    "import numpy as np\n",
+    "\n",
+    "# Test: Unicode\n",
+    "\n",
+    "\n",
+    "def c_star(x, β, γ):\n",
+    "\n",
+    "    return (1 - β ** (1 / γ)) * x\n"
    ]
   }
  ],
@@ -44,13 +53,13 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.9"
+   "version": "3.8.8"
   },
   "source_map": [
    11,
    15
   ]
  },
  "nbformat": 4,
- "nbformat_minor": 4
+ "nbformat_minor": 5
 }

tests/test_mystnb_features/test_codecell_file.xml

@@ -8,3 +8,10 @@
                     # flake8: noqa
 
                     import numpy as np
+
+                    # Test: Unicode
+
+
+                    def c_star(x, β, γ):
+
+                        return (1 - β ** (1 / γ)) * x

tests/test_mystnb_features/test_codecell_file_warnings.ipynb

@@ -2,6 +2,7 @@
  "cells": [
   {
    "cell_type": "markdown",
+   "id": "0d5a00cd",
    "metadata": {},
    "source": [
     "# a title"
@@ -10,14 +11,22 @@
   {
    "cell_type": "code",
    "execution_count": 1,
+   "id": "0b392fbd",
    "metadata": {
-    "file": "mystnb_codecell_file.py"
+    "load": "mystnb_codecell_file.py"
    },
    "outputs": [],
    "source": [
     "# flake8: noqa\n",
     "\n",
-    "import numpy as np\n"
+    "import numpy as np\n",
+    "\n",
+    "# Test: Unicode\n",
+    "\n",
+    "\n",
+    "def c_star(x, β, γ):\n",
+    "\n",
+    "    return (1 - β ** (1 / γ)) * x\n"
    ]
   }
  ],
@@ -44,13 +53,13 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.5"
+   "version": "3.8.8"
   },
   "source_map": [
    11,
    15
   ]
  },
  "nbformat": 4,
- "nbformat_minor": 4
+ "nbformat_minor": 5
 }

tests/test_mystnb_features/test_codecell_file_warnings.xml

@@ -8,3 +8,10 @@
                     # flake8: noqa
 
                     import numpy as np
+
+                    # Test: Unicode
+
+
+                    def c_star(x, β, γ):
+
+                        return (1 - β ** (1 / γ)) * x