compatibility with mathics-core PR 986

.github/workflows/ubuntu.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

mathics_django/doc/django_doc.py

@@ -6,20 +6,23 @@
 
 import pickle
 import re
+from typing import Optional
 
 from django.utils.safestring import mark_safe
 from mathics import settings
 from mathics.doc.common_doc import (
     DocChapter,
     DocGuideSection,
+    DocSection,
+    DocSubsection,
     DocTest,
     DocTests,
     DocText,
-    Documentation,
+    DocumentationEntry,
+    MathicsMainDocumentation,
     Tests,
-    XMLDoc,
-    gather_tests,
     get_results_by_test,
+    parse_docstring_to_DocumentationEntry_items,
     sorted_chapters,
 )
 from mathics.doc.utils import slugify
@@ -28,21 +31,21 @@
 from mathics_django.settings import get_doctest_html_data_path
 
 # FIXME: remove globalness
+doctest_html_data_path = get_doctest_html_data_path(should_be_readable=True)
 try:
-    doctest_html_data_path = get_doctest_html_data_path(should_be_readable=True)
     with open(doctest_html_data_path, "rb") as doctest_html_data_file:
         doc_data = pickle.load(doctest_html_data_file)
 except IOError:
     print(f"Trouble reading Doc file {doctest_html_data_path}")
     doc_data = {}
 
 
-class DjangoDocElement(object):
+class DjangoDocElement:
     def href(self, ajax=False):
         if ajax:
-            return "javascript:loadDoc('%s')" % self.get_uri()
+            return f"javascript:loadDoc('{self.get_uri()}')"
         else:
-            return "/doc%s" % self.get_uri()
+            return f"/doc{self.get_uri()}"
 
     def get_prev(self):
         return self.get_prev_next()[0]
@@ -64,11 +67,11 @@ def get_title_html(self):
         return mark_safe(escape_html(self.title, single_line=True))
 
 
-class DjangoDocumentation(Documentation, DjangoDocElement):
+class DjangoDocumentationMixin(DjangoDocElement):
     def __str__(self):
         return "\n\n\n".join(str(part) for part in self.parts)
 
-    def get_tests(self):
+    def ___get_tests(self):
         for part in self.parts:
             for chapter in sorted_chapters(part.chapters):
                 tests = chapter.doc.get_tests()
@@ -117,6 +120,23 @@ def get_tests(self):
             pass
         return
 
+
+class MathicsDjangoDocumentation(MathicsMainDocumentation, DjangoDocElement):
+    def __init__(self, want_sorting=True):
+
+        self.chapter_class = DjangoDocChapter
+        self.doc_dir = settings.DOC_DIR
+        self.doc_class = DjangoDocumentationEntry
+        self.guide_section_class = DjangoDocGuideSection
+        self.part_class = DjangoDocPart
+        self.section_class = DjangoDocSection
+        self.subsection_class = DjangoDocSubsection
+        # Initialize the superclass
+        super().__init__(want_sorting)
+        # Now, let's load the documentation
+        self.load_documentation_sources()
+        self.title = "Mathics Documentation"
+
     def get_uri(self) -> str:
         return "/"
 
@@ -161,7 +181,7 @@ def search_sections(section, result):
             for chapter in sorted_chapters(part.chapters):
                 if matches(chapter.title):
                     result.append((False, chapter))
-                for section in chapter.sections:
+                for section in chapter.all_sections:
                     if matches(section.title):
                         if not isinstance(section, DjangoDocGuideSection):
                             result.append((section.title == query, section))
@@ -174,39 +194,12 @@ def search_sections(section, result):
         return sorted_results
 
 
-class MathicsDjangoDocumentation(DjangoDocumentation):
-    def __init__(self, want_sorting=True):
-
-        self.doc_chapter_fn = DjangoDocChapter
-        self.doc_dir = settings.DOC_DIR
-        self.doc_fn = DjangoDoc
-        self.doc_guide_section_fn = DjangoDocGuideSection
-        self.doc_part_fn = DjangoDocPart
-        self.doc_section_fn = DjangoDocSection
-        self.doc_subsection_fn = DjangoDocSubsection
-        self.parts = []
-        self.parts_by_slug = {}
-        self.title = "Overview"
-
-        self.gather_doctest_data()
-
-
-class DjangoDoc(XMLDoc):
-    def __init__(self, doc, title, section):
-        self.title = title
-        if section:
-            chapter = section.chapter
-            part = chapter.part
-            # Note: we elide section.title
-            key_prefix = (part.title, chapter.title, title)
-        else:
-            key_prefix = None
-
-        self.rawdoc = doc
-        self.items = gather_tests(
-            self.rawdoc, DjangoDocTests, DjangoDocTest, DjangoDocText, key_prefix
-        )
-        return
+class DjangoDocumentationEntry(DocumentationEntry):
+    def __init__(self, doc_str: str, title: str, section: Optional["DjangoDocSection"]):
+        self.docTest_collection_class = DjangoDocTests
+        self.docTest_class = DjangoDocTest
+        self.docText_class = DjangoDocText
+        super().__init__(doc_str, title, section)
 
     def __str__(self):
         return "\n".join(str(item) for item in self.items)
@@ -218,7 +211,6 @@ def get_tests(self):
         return tests
 
     def html(self):
-        counters = {}
         items = [item for item in self.items if not item.is_private()]
         title_line = self.title + "\n"
         if len(items) and items[0].text.startswith(title_line):
@@ -227,7 +219,7 @@ def html(self):
             # Or that is the intent. This code is a bit hacky.
             items[0].text = items[0].text[len(title_line) :]
 
-        text = "\n".join(item.html(counters) for item in items if not item.is_private())
+        text = "\n".join(item.html() for item in items if not item.is_private())
         if text == "":
             # HACK ALERT if text is "" we may have missed some test markup.
             return mark_safe(escape_html(self.rawdoc))
@@ -268,7 +260,7 @@ def get_collection(self):
         """Return a list of parts in this doc"""
         return self.doc.parts
 
-    def html(self, counters=None):
+    def html(self):
         if len(self.tests) == 0:
             return "\n"
         return '<ul class="tests">%s</ul>' % (
@@ -281,7 +273,7 @@ def get_uri(self) -> str:
         return f"/{self.slug}/"
 
 
-class DjangoDocSection(DjangoDocElement):
+class DjangoDocSection(DocSection, DjangoDocElement):
     """An object for a Django Documented Section.
     A Section is part of a Chapter. It can contain subsections.
     """
@@ -308,13 +300,12 @@ def __init__(
 
         if text.count("<dl>") != text.count("</dl>"):
             raise ValueError(
-                "Missing opening or closing <dl> tag in "
-                "{} documentation".format(title)
+                f"Missing opening or closing <dl> tag in {title} documentation"
             )
 
         # Needs to come after self.chapter is initialized since
-        # XMLDoc uses self.chapter.
-        self.doc = DjangoDoc(text, title, self)
+        # DocumentationEntry uses self.chapter.
+        self.doc = DjangoDocumentationEntry(text, title, self)
 
         chapter.sections_by_slug[self.slug] = self
 
@@ -324,7 +315,7 @@ def __str__(self):
     def get_collection(self):
         """Return a list of subsections for this section that this section belongs
         to."""
-        return self.chapter.sections
+        return self.chapter.all_sections
 
     def html_data(self):
         indices = set()
@@ -353,7 +344,7 @@ def __init__(
         self, chapter: str, title: str, text: str, submodule, installed: bool = True
     ):
         self.chapter = chapter
-        self.doc = DjangoDoc(text, title, None)
+        self.doc = DjangoDocumentationEntry(text, title, None)
         self.in_guide = False
         self.installed = installed
         self.slug = slugify(title)
@@ -370,8 +361,7 @@ def __init__(
 
         if text.count("<dl>") != text.count("</dl>"):
             raise ValueError(
-                "Missing opening or closing <dl> tag in "
-                "{} documentation".format(title)
+                f"Missing opening or closing <dl> tag in {title} documentation"
             )
         # print("YYY Adding section", title)
         chapter.sections_by_slug[self.slug] = self
@@ -381,7 +371,7 @@ def get_uri(self) -> str:
         return f"/{self.chapter.part.slug}/{self.chapter.slug}/guide/"
 
 
-class DjangoDocSubsection(DjangoDocElement):
+class DjangoDocSubsection(DocSubsection, DjangoDocElement):
     """An object for a Django Documented Subsection.
     A Subsection is part of a Section.
     """
@@ -417,13 +407,18 @@ def __init__(
         mathics/builtin/colors/named-colors.py we have the "section"
         name for the class Read (the subsection) inside it.
         """
-
+        super().__init__(
+            chapter, section, title, text, operator, installed, in_guide, summary_text
+        )
+        self.doc = DjangoDocumentationEntry(text, title, section)
+        return
+        # Check if any of this is actually needed.
         title_summary_text = re.split(" -- ", title)
         n = len(title_summary_text)
         self.title = title_summary_text[0] if n > 0 else ""
         self.summary_text = title_summary_text[1] if n > 1 else summary_text
 
-        self.doc = DjangoDoc(text, title, section)
+        self.doc = DjangoDocumentationEntry(text, title, section)
         self.chapter = chapter
         self.installed = installed
         self.operator = operator
@@ -451,8 +446,7 @@ def __init__(
 
         if text.count("<dl>") != text.count("</dl>"):
             raise ValueError(
-                "Missing opening or closing <dl> tag in "
-                "{} documentation".format(title)
+                f"Missing opening or closing <dl> tag in {title} documentation"
             )
         self.section.subsections_by_slug[self.slug] = self
 
@@ -521,7 +515,7 @@ def html(self) -> str:
 
 
 class DjangoDocTests(DocTests):
-    def html(self, counters=None):
+    def html(self):
         if len(self.tests) == 0:
             return "\n"
         return '<ul class="tests">%s</ul>' % (
@@ -532,6 +526,6 @@ def html(self, counters=None):
 
 
 class DjangoDocText(DocText):
-    def html(self, counters=None) -> str:
-        result = escape_html(self.text, counters=counters)
+    def html(self) -> str:
+        result = escape_html(self.text)
         return result

mathics_django/docpipeline.py

@@ -598,9 +598,6 @@ def main():
     if args.logfilename:
         logfile = open(args.logfilename, "wt")
 
-    global documentation
-    documentation = MathicsDjangoDocumentation()
-
     # LoadModule Mathics3 modules
     if args.pymathics:
         for module_name in args.pymathics.split(","):
@@ -614,7 +611,10 @@ def main():
             else:
                 print(f"Mathics3 Module {module_name} loaded")
 
-    documentation.gather_doctest_data()
+    # MathicsDjangoDocumentation load the documentation automatically.
+    # It must be loaded after loading modules.
+    global documentation
+    documentation = MathicsDjangoDocumentation()
 
     if args.sections:
         sections = set(args.sections.split(","))

mathics_django/settings.py

@@ -74,7 +74,7 @@ def get_bool_from_environment(env_var: str, default_value: str):
 )
 
 # We need another version as a fallback, and that is distributed with the
-# package. It is note user writable and not in the user space.
+# package. It is not user writable and not in the user space.
 DOC_SYSTEM_HTML_DATA_PATH = os.environ.get(
     "DOC_SYSTEM_HTML_DATA_PATH", osp.join(ROOT_DIR, "doc", "doc_html_data.pcl")
 )

mathics_django/web/controllers/doc.py

@@ -8,7 +8,8 @@
 from django.core.handlers.wsgi import WSGIRequest
 from django.http import Http404, HttpResponse
 from django.shortcuts import render
-from mathics.eval.pymathics import pymathics_modules
+from mathics.doc.common_doc import get_module_doc
+from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules
 
 from mathics_django.doc import documentation
 from mathics_django.doc.django_doc import (
@@ -21,12 +22,37 @@
 
 DocResponse = Union[HttpResponse, JsonResponse]
 
+seen_pymathics_modules = copy(pymathics_modules)
+
 
 def check_for_pymathics_load():
+    global seen_pymathics_modules
     if seen_pymathics_modules != pymathics_modules:
-        # print("XXX refresh pymathics doc")
-        global documentation
-        documentation = MathicsDjangoDocumentation()
+        print("XXX refresh pymathics doc", pymathics_modules)
+        new_modules = pymathics_modules - seen_pymathics_modules
+        for new_module in new_modules:
+            mathics3_module_part = documentation.parts_by_slug.get(
+                "mathics3-modules", None
+            )
+            # If this is the first loaded module, we need to create the Part
+            if mathics3_module_part is None:
+                mathics3_module_part = self.doc_part(
+                    "Mathics3 Modules",
+                    pymathics_modules,
+                    pymathics_builtins_by_module,
+                    True,
+                )
+                seen_pymathics_modules = copy(pymathics_modules)
+                return
+
+            # The "Mathics3 modules" part already exists. Lets add the new chapter.
+            chapter = documentation.doc_chapter(
+                new_module, mathics3_module_part, pymathics_builtins_by_module
+            )
+            mathics3_module_part.chapters.append(chapter)
+
+        seen_pymathics_modules = copy(pymathics_modules)
+        pass
 
 
 def doc(request: WSGIRequest, ajax: bool = False) -> DocResponse:
@@ -44,7 +70,9 @@ def doc(request: WSGIRequest, ajax: bool = False) -> DocResponse:
 
 def doc_chapter(request: WSGIRequest, part, chapter, ajax: bool = False) -> DocResponse:
     """
-    Produces HTML via jinja templating for a chapter. Some examples of Chapters:
+    Produces HTML via jinja templating for a chapter. Some examples of
+    Chapters:
+
     * Introduction (in part Manual)
     * Procedural Programming (in part Reference of Built-in Symbols)
     """
@@ -87,9 +115,6 @@ def doc_part(request: WSGIRequest, part, ajax: bool = False) -> DocResponse:
     )
 
 
-seen_pymathics_modules = copy(pymathics_modules)
-
-
 def doc_search(request: WSGIRequest) -> DocResponse:
     check_for_pymathics_load()
     query = request.GET.get("query", "")