From 451f4d7afde1fa7f212cc835af9245310a75289e Mon Sep 17 00:00:00 2001
From: rocky <rb@dustyfeet.com>
Date: Fri, 19 Jan 2024 12:46:21 -0500
Subject: [PATCH 01/23] Bump copyright

---
 mathics_django/web/templates/about.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mathics_django/web/templates/about.html b/mathics_django/web/templates/about.html
index 45a12a6c3..e390df2fc 100644
--- a/mathics_django/web/templates/about.html
+++ b/mathics_django/web/templates/about.html
@@ -165,7 +165,7 @@ <h1>Connection Information</h1>
 
 			  <p>Mathics3 is a general-purpose computer algebra system.</p>
 
-			  <p>Copyright (C) 2021-2023 The Mathics3 Team<p><br>
+			  <p>Copyright (C) 2021-2024 The Mathics3 Team<p><br>
 
 			  This program comes with ABSOLUTELY NO WARRANTY;
 			    This is free software, and you are welcome to redistribute it

From a2be2e022577d398b6174c9e863707639630f51a Mon Sep 17 00:00:00 2001
From: rocky <rb@dustyfeet.com>
Date: Fri, 2 Feb 2024 16:44:08 -0500
Subject: [PATCH 02/23] Changes to go track changes in core PR #984

---
 mathics_django/doc/__init__.py        |   4 +-
 mathics_django/doc/django_doc.py      | 162 +++------
 mathics_django/docpipeline.py         | 481 +++-----------------------
 mathics_django/settings.py            |   2 +-
 mathics_django/web/controllers/doc.py |  40 ++-
 5 files changed, 127 insertions(+), 562 deletions(-)

diff --git a/mathics_django/doc/__init__.py b/mathics_django/doc/__init__.py
index 7e1bb0b8b..0ec4b2531 100644
--- a/mathics_django/doc/__init__.py
+++ b/mathics_django/doc/__init__.py
@@ -8,6 +8,6 @@
 import_and_load_builtins()
 
 # FIXME: should we really do this here?
-from mathics_django.doc.django_doc import MathicsDjangoDocumentation
+from mathics_django.doc.django_doc import DjangoDocumentation
 
-documentation = MathicsDjangoDocumentation()
+documentation = DjangoDocumentation()
diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index ed676d06c..392013669 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -11,12 +11,13 @@
 from mathics import settings
 from mathics.doc.common_doc import (
     DocChapter,
-    DocGuideSection,
+    DocPart,
+    DocSection,
+    DocSubsection,
     DocTest,
     DocTests,
     DocText,
     Documentation,
-    Tests,
     XMLDoc,
     gather_tests,
     get_results_by_test,
@@ -28,8 +29,8 @@
 from mathics_django.settings import get_doctest_html_data_path
 
 # FIXME: remove globalness
+doctest_html_data_path = get_doctest_html_data_path(should_be_readable=True)
 try:
-    doctest_html_data_path = get_doctest_html_data_path(should_be_readable=True)
     with open(doctest_html_data_path, "rb") as doctest_html_data_file:
         doc_data = pickle.load(doctest_html_data_file)
 except IOError:
@@ -37,12 +38,16 @@
     doc_data = {}
 
 
-class DjangoDocElement(object):
+class DjangoDocElement:
+    """
+    Adds some HTML functions onto existing Django Document Elements.
+    """
+
     def href(self, ajax=False):
         if ajax:
-            return "javascript:loadDoc('%s')" % self.get_uri()
+            return f"javascript:loadDoc('{self.get_uri()}')"
         else:
-            return "/doc%s" % self.get_uri()
+            return f"/doc{self.get_uri()}"
 
     def get_prev(self):
         return self.get_prev_next()[0]
@@ -65,58 +70,27 @@ def get_title_html(self):
 
 
 class DjangoDocumentation(Documentation, DjangoDocElement):
+    def __init__(self):
+        super(DjangoDocumentation, self).__init__()
+        self.doc_class = DjangoDoc
+        self.doc_dir = settings.DOC_DIR
+        self.chapter_class = DjangoDocChapter
+        self.guide_section_class = DjangoDocGuideSection
+        self.part_class = DjangoDocPart
+        self.section_class = DjangoDocSection
+        self.subsection_class = DjangoDocSubsection
+
+        self.gather_doctest_data()
+        self.doctest_latex_pcl_path = settings.DOCTEST_LATEX_DATA_PCL
+        self.pymathics_doc_loaded = False
+        self.doc_data_file = settings.get_doctest_latex_data_path(
+            should_be_readable=True
+        )
+        self.title = "Overview"
+
     def __str__(self):
         return "\n\n\n".join(str(part) for part in self.parts)
 
-    def get_tests(self):
-        for part in self.parts:
-            for chapter in sorted_chapters(part.chapters):
-                tests = chapter.doc.get_tests()
-                if tests:
-                    yield Tests(part.title, chapter.title, "", tests)
-                for section in chapter.sections:
-                    if section.installed:
-                        if isinstance(section, DocGuideSection):
-                            for docsection in section.subsections:
-                                for docsubsection in docsection.subsections:
-                                    # FIXME: Something is weird here
-                                    # where tests for subsection items
-                                    # appear not as a collection but
-                                    # individually and need to be
-                                    # iterated below. Probably some
-                                    # other code is faulty and when
-                                    # fixed the below loop and
-                                    # collection into doctest_list[]
-                                    # will be removed.
-                                    doctest_list = []
-                                    index = 1
-                                    for doctests in docsubsection.items:
-                                        doctest_list += list(doctests.get_tests())
-                                        for test in doctest_list:
-                                            test.index = index
-                                            index += 1
-
-                                    if doctest_list:
-                                        yield Tests(
-                                            section.chapter.part.title,
-                                            section.chapter.title,
-                                            docsubsection.title,
-                                            doctest_list,
-                                        )
-                        else:
-                            tests = section.doc.get_tests()
-                            if tests:
-                                yield Tests(
-                                    part.title, chapter.title, section.title, tests
-                                )
-                                pass
-                            pass
-                        pass
-                    pass
-                pass
-            pass
-        return
-
     def get_uri(self) -> str:
         return "/"
 
@@ -174,33 +148,17 @@ def search_sections(section, result):
         return sorted_results
 
 
-class MathicsDjangoDocumentation(DjangoDocumentation):
-    def __init__(self, want_sorting=True):
-
-        self.doc_chapter_fn = DjangoDocChapter
-        self.doc_dir = settings.DOC_DIR
-        self.doc_fn = DjangoDoc
-        self.doc_guide_section_fn = DjangoDocGuideSection
-        self.doc_part_fn = DjangoDocPart
-        self.doc_section_fn = DjangoDocSection
-        self.doc_subsection_fn = DjangoDocSubsection
-        self.parts = []
-        self.parts_by_slug = {}
-        self.title = "Overview"
-
-        self.gather_doctest_data()
-
-
 class DjangoDoc(XMLDoc):
-    def __init__(self, doc, title, section):
+    def __init__(self, doc, title, section, key_prefix=None):
         self.title = title
-        if section:
-            chapter = section.chapter
-            part = chapter.part
-            # Note: we elide section.title
-            key_prefix = (part.title, chapter.title, title)
-        else:
-            key_prefix = None
+        if key_prefix is None:
+            if section is not None:
+                chapter = section.chapter
+                part = chapter.part
+                # Note: we elide section.title
+                key_prefix = (part.title, chapter.title, title)
+            else:
+                key_prefix = None
 
         self.rawdoc = doc
         self.items = gather_tests(
@@ -218,7 +176,6 @@ def get_tests(self):
         return tests
 
     def html(self):
-        counters = {}
         items = [item for item in self.items if not item.is_private()]
         title_line = self.title + "\n"
         if len(items) and items[0].text.startswith(title_line):
@@ -227,7 +184,7 @@ def html(self):
             # Or that is the intent. This code is a bit hacky.
             items[0].text = items[0].text[len(title_line) :]
 
-        text = "\n".join(item.html(counters) for item in items if not item.is_private())
+        text = "\n".join(item.html() for item in items if not item.is_private())
         if text == "":
             # HACK ALERT if text is "" we may have missed some test markup.
             return mark_safe(escape_html(self.rawdoc))
@@ -247,28 +204,16 @@ def get_uri(self) -> str:
         return f"/{self.part.slug}/{self.slug}/"
 
 
-class DjangoDocPart(DjangoDocElement):
+class DjangoDocPart(DocPart, DjangoDocElement):
     def __init__(self, doc, title, is_reference=False):
-        self.doc = doc
-        self.title = title
-        self.slug = slugify(title)
-        self.chapters = []
-        self.chapters_by_slug = {}
-        self.is_reference = is_reference
-        self.is_appendix = False
-        doc.parts_by_slug[self.slug] = self
-
-    def __str__(self):
-        return "%s\n\n%s" % (
-            self.title,
-            "\n".join(str(chapter) for chapter in sorted_chapters(self.chapters)),
-        )
+        super(DjangoDocPart, self).__init__(doc, title, is_reference)
+        self.chapter_class = DjangoDocChapter
 
     def get_collection(self):
         """Return a list of parts in this doc"""
         return self.doc.parts
 
-    def html(self, counters=None):
+    def html(self):
         if len(self.tests) == 0:
             return "\n"
         return '<ul class="tests">%s</ul>' % (
@@ -281,7 +226,7 @@ def get_uri(self) -> str:
         return f"/{self.slug}/"
 
 
-class DjangoDocSection(DjangoDocElement):
+class DjangoDocSection(DocSection, DjangoDocElement):
     """An object for a Django Documented Section.
     A Section is part of a Chapter. It can contain subsections.
     """
@@ -308,8 +253,7 @@ def __init__(
 
         if text.count("<dl>") != text.count("</dl>"):
             raise ValueError(
-                "Missing opening or closing <dl> tag in "
-                "{} documentation".format(title)
+                f"Missing opening or closing <dl> tag in {title} documentation"
             )
 
         # Needs to come after self.chapter is initialized since
@@ -342,7 +286,7 @@ def get_uri(self) -> str:
         return f"/{self.chapter.part.slug}/{self.chapter.slug}/{self.slug}/"
 
 
-class DjangoDocGuideSection(DjangoDocSection):
+class DjangoDocGuideSection(DjangoDocSection, DjangoDocElement):
     """An object for a Django Documented Guide Section.
     A Guide Section is part of a Chapter. "Colors" or "Special Functions"
     are examples of Guide Sections, and each contains a number of Sections.
@@ -370,8 +314,7 @@ def __init__(
 
         if text.count("<dl>") != text.count("</dl>"):
             raise ValueError(
-                "Missing opening or closing <dl> tag in "
-                "{} documentation".format(title)
+                f"Missing opening or closing <dl> tag in {title} documentation"
             )
         # print("YYY Adding section", title)
         chapter.sections_by_slug[self.slug] = self
@@ -381,7 +324,7 @@ def get_uri(self) -> str:
         return f"/{self.chapter.part.slug}/{self.chapter.slug}/guide/"
 
 
-class DjangoDocSubsection(DjangoDocElement):
+class DjangoDocSubsection(DocSubsection, DjangoDocElement):
     """An object for a Django Documented Subsection.
     A Subsection is part of a Section.
     """
@@ -451,8 +394,7 @@ def __init__(
 
         if text.count("<dl>") != text.count("</dl>"):
             raise ValueError(
-                "Missing opening or closing <dl> tag in "
-                "{} documentation".format(title)
+                f"Missing opening or closing <dl> tag in {title} documentation"
             )
         self.section.subsections_by_slug[self.slug] = self
 
@@ -521,7 +463,7 @@ def html(self) -> str:
 
 
 class DjangoDocTests(DocTests):
-    def html(self, counters=None):
+    def html(self):
         if len(self.tests) == 0:
             return "\n"
         return '<ul class="tests">%s</ul>' % (
@@ -532,6 +474,6 @@ def html(self, counters=None):
 
 
 class DjangoDocText(DocText):
-    def html(self, counters=None) -> str:
-        result = escape_html(self.text, counters=counters)
+    def html(self) -> str:
+        result = escape_html(self.text)
         return result
diff --git a/mathics_django/docpipeline.py b/mathics_django/docpipeline.py
index 04a1854f6..7d0b96418 100644
--- a/mathics_django/docpipeline.py
+++ b/mathics_django/docpipeline.py
@@ -1,412 +1,42 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# FIXME: combine with same thing in Mathics Django
+# FIXME: combine with same thing in Mathics core
 """
 Does 2 things which can either be done independently or
 as a pipeline:
 
-1. Extracts tests and runs them from static mdoc files and docstrings from Mathics
-   built-in functions
+1. Extracts tests and runs them from static mdoc files and docstrings from
+   Mathics built-in functions
 2. Creates/updates internal documentation data
 """
 
-import os
-import os.path as osp
 import pickle
-import re
-import sys
 from argparse import ArgumentParser
 from datetime import datetime
 
 import mathics
-from mathics import version_string
+import mathics.docpipeline as md
 from mathics.core.definitions import Definitions
-from mathics.core.evaluation import Evaluation, Output
 from mathics.core.load_builtin import (
     builtins_by_module,
     builtins_dict,
     import_and_load_builtins,
 )
-from mathics.core.parser import MathicsSingleLineFeeder
+from mathics.docpipeline import (
+    MAX_TESTS,
+    create_output,
+    open_ensure_dir,
+    print_and_log,
+    test_all,
+    test_chapters,
+    test_sections,
+    write_doctest_data,
+)
 from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule
 
-from mathics_django.doc import MathicsDjangoDocumentation
+from mathics_django.doc import DjangoDocumentation
 from mathics_django.settings import get_doctest_html_data_path
 
-builtins = builtins_dict(builtins_by_module)
-
-
-class TestOutput(Output):
-    def max_stored_size(self):
-        return None
-
-
-sep = "-" * 70 + "\n"
-
-# Global variables
-definitions = None
-documentation = None
-check_partial_enlapsed_time = False
-logfile = None
-
-
-MAX_TESTS = 100000  # Number than the total number of tests
-
-
-def print_and_log(*args):
-    a = [a.decode("utf-8") if isinstance(a, bytes) else a for a in args]
-    string = "".join(a)
-    print(string)
-    if logfile:
-        logfile.write(string)
-
-
-def compare(result, wanted) -> bool:
-    if result == wanted:
-        return True
-
-    if result is None or wanted is None:
-        return False
-    result = result.splitlines()
-    wanted = wanted.splitlines()
-    if result == [] and wanted == ["#<--#"]:
-        return True
-    if len(result) != len(wanted):
-        return False
-    for r, w in zip(result, wanted):
-        wanted_re = re.escape(w.strip())
-        wanted_re = wanted_re.replace("\\.\\.\\.", ".*?")
-        wanted_re = "^%s$" % wanted_re
-        if not re.match(wanted_re, r.strip()):
-            return False
-    return True
-
-
-stars = "*" * 10
-
-
-def test_case(test, tests, index=0, subindex=0, quiet=False, section=None) -> bool:
-    global check_partial_enlapsed_time
-    test, wanted_out, wanted = test.test, test.outs, test.result
-
-    def fail(why):
-        part, chapter, section = tests.part, tests.chapter, tests.section
-        print_and_log(
-            f"""{sep}Test failed: {section} in {part} / {chapter}
-{part}
-{why}
-""".encode(
-                "utf-8"
-            )
-        )
-        return False
-
-    if not quiet:
-        if section:
-            print(f"{stars} {tests.chapter} / {section} {stars}".encode("utf-8"))
-        print(f"{index:4d} ({subindex:2d}): TEST {test}".encode("utf-8"))
-
-    feeder = MathicsSingleLineFeeder(test, "<test>")
-    evaluation = Evaluation(definitions, catch_interrupt=False, output=TestOutput())
-    try:
-        time_parsing = datetime.now()
-        query = evaluation.parse_feeder(feeder)
-        if check_partial_enlapsed_time:
-            print("   parsing took", datetime.now() - time_parsing)
-        if query is None:
-            # parsed expression is None
-            result = None
-            out = evaluation.out
-        else:
-            result = evaluation.evaluate(query)
-            if check_partial_enlapsed_time:
-                print("   evaluation took", datetime.now() - time_parsing)
-            out = result.out
-            result = result.result
-    except Exception as exc:
-        fail("Exception %s" % exc)
-        info = sys.exc_info()
-        sys.excepthook(*info)
-        return False
-
-    time_comparing = datetime.now()
-    comparison_result = compare(result, wanted)
-
-    if check_partial_enlapsed_time:
-        print("   comparison took ", datetime.now() - time_comparing)
-    if not comparison_result:
-        print("result =!=wanted")
-        fail_msg = "Result: %s\nWanted: %s" % (result, wanted)
-        if out:
-            fail_msg += "\nAdditional output:\n"
-            fail_msg += "\n".join(str(o) for o in out)
-        return fail(fail_msg)
-    output_ok = True
-    time_comparing = datetime.now()
-    if len(wanted_out) == 1 and wanted_out[0].text == "...":
-        # If we have ... don't check
-        pass
-    elif len(out) != len(wanted_out):
-        # Mismatched number of output lines and we don't have "..."
-        output_ok = False
-    else:
-        # Need to check all output line by line
-        for got, wanted in zip(out, wanted_out):
-            if not got == wanted and wanted.text != "...":
-                output_ok = False
-                break
-    if check_partial_enlapsed_time:
-        print("   comparing messages took ", datetime.now() - time_comparing)
-    if not output_ok:
-        return fail(
-            "Output:\n%s\nWanted:\n%s"
-            % ("\n".join(str(o) for o in out), "\n".join(str(o) for o in wanted_out))
-        )
-    return True
-
-
-def test_tests(
-    tests,
-    index,
-    quiet=False,
-    stop_on_failure=False,
-    start_at=0,
-    max_tests=MAX_TESTS,
-    excludes=[],
-):
-    definitions.reset_user_definitions()
-    total = failed = skipped = 0
-    failed_symbols = set()
-    section = tests.section
-    if section in excludes:
-        return total, failed, len(tests.tests), failed_symbols, index
-    count = 0
-    for subindex, test in enumerate(tests.tests):
-        index += 1
-        if test.ignore:
-            continue
-        if index < start_at:
-            skipped += 1
-            continue
-        elif count >= max_tests:
-            break
-
-        total += 1
-        count += 1
-        if not test_case(test, tests, index, subindex + 1, quiet, section):
-            failed += 1
-            failed_symbols.add((tests.part, tests.chapter, tests.section))
-            if stop_on_failure:
-                break
-
-        section = None
-    return total, failed, skipped, failed_symbols, index
-
-
-# FIXME: move this to common routine
-def create_output(tests, doc_data, format="xml"):
-    definitions.reset_user_definitions()
-    for test in tests.tests:
-        if test.private:
-            continue
-        key = test.key
-        evaluation = Evaluation(
-            definitions, format=format, catch_interrupt=True, output=TestOutput()
-        )
-        try:
-            result = evaluation.parse_evaluate(test.test)
-        except:  # noqa
-            result = None
-        if result is None:
-            result = []
-        else:
-            result = [result.get_data()]
-        doc_data[key] = {
-            "query": test.test,
-            "results": result,
-        }
-
-
-def test_chapters(
-    chapters: set,
-    quiet=False,
-    stop_on_failure=False,
-    generate_output=False,
-    reload=False,
-    want_sorting=False,
-):
-    if documentation is None:
-        print_and_log("documentation is not loaded.")
-        return
-    failed = 0
-    index = 0
-    chapter_names = ", ".join(chapters)
-    print(f"Testing chapter(s): {chapter_names}")
-    output_data = load_doc_data() if reload else {}
-    prev_key = []
-    for tests in documentation.get_tests():
-        if tests.chapter in sorted(chapters):
-            for test in tests.tests:
-                key = list(test.key)[1:-1]
-                if prev_key != key:
-                    prev_key = key
-                    print(f'Testing section: {" / ".join(key)}')
-                    index = 0
-                if test.ignore:
-                    continue
-                index += 1
-                if not test_case(test, tests, index, quiet=quiet):
-                    failed += 1
-                    if stop_on_failure:
-                        break
-            if generate_output and failed == 0:
-                create_output(tests, output_data)
-
-    print()
-    if index == 0:
-        print_and_log(f"No chapters found named {chapter_names}.")
-    elif failed > 0:
-        if not (keep_going and format == "xml"):
-            print_and_log("%d test%s failed." % (failed, "s" if failed != 1 else ""))
-    else:
-        print_and_log("All tests passed.")
-
-
-def test_sections(
-    sections: set,
-    quiet=False,
-    stop_on_failure=False,
-    generate_output=False,
-    reload=False,
-    want_sorting=False,
-):
-    if documentation is None:
-        print_and_log("documentation is not loaded.")
-        return
-    failed = 0
-    index = 0
-    section_names = ", ".join(sections)
-    print(f"Testing section(s): {section_names}")
-    sections |= {"$" + s for s in sections}
-    output_data = load_doc_data() if reload else {}
-    prev_key = []
-    for tests in documentation.get_tests():
-        if tests.section in sections:
-            for test in tests.tests:
-                key = list(test.key)[1:-1]
-                if prev_key != key:
-                    prev_key = key
-                    print(f'Testing section: {" / ".join(key)}')
-                    index = 0
-                if test.ignore:
-                    continue
-                index += 1
-                if not test_case(test, tests, index, quiet=quiet):
-                    failed += 1
-                    if stop_on_failure:
-                        break
-            if generate_output and failed == 0:
-                create_output(tests, output_data)
-
-    print()
-    if index == 0:
-        print_and_log(f"No sections found named {section_names}.")
-    elif failed > 0:
-        if not (keep_going and format == "xml"):
-            print_and_log("%d test%s failed." % (failed, "s" if failed != 1 else ""))
-    else:
-        print_and_log("All tests passed.")
-    if generate_output and (failed == 0 or keep_going):
-        save_doctest_data(output_data)
-
-
-def open_ensure_dir(f, *args, **kwargs):
-    try:
-        return open(f, *args, **kwargs)
-    except (IOError, OSError):
-        d = osp.dirname(f)
-        if d and not osp.exists(d):
-            os.makedirs(d)
-        return open(f, *args, **kwargs)
-
-
-def test_all(
-    quiet=False,
-    generate_output=False,
-    stop_on_failure=False,
-    start_at=0,
-    count=MAX_TESTS,
-    texdatafolder=None,
-    doc_even_if_error=False,
-    excludes=[],
-    want_sorting=False,
-):
-    if not quiet:
-        print(f"Testing {version_string}")
-
-    if documentation is None:
-        print_and_log("documentation is not loaded.")
-        return
-
-    try:
-        index = 0
-        total = failed = skipped = 0
-        failed_symbols = set()
-        output_data = {}
-        for tests in documentation.get_tests():
-            sub_total, sub_failed, sub_skipped, symbols, index = test_tests(
-                tests,
-                index,
-                quiet=quiet,
-                stop_on_failure=stop_on_failure,
-                start_at=start_at,
-                max_tests=count,
-                excludes=excludes,
-            )
-            if generate_output:
-                create_output(tests, output_data)
-            total += sub_total
-            failed += sub_failed
-            skipped += sub_skipped
-            failed_symbols.update(symbols)
-            if sub_failed and stop_on_failure:
-                break
-            if total >= count:
-                break
-        builtin_total = len(builtins)
-    except KeyboardInterrupt:
-        print("\nAborted.\n")
-        return
-
-    if failed > 0:
-        print(sep)
-    if count == MAX_TESTS:
-        print_and_log(
-            "%d Tests for %d built-in symbols, %d passed, %d failed, %d skipped."
-            % (total, builtin_total, total - failed - skipped, failed, skipped)
-        )
-    else:
-        print_and_log(
-            "%d Tests, %d passed, %d failed, %d skipped."
-            % (total, total - failed, failed, skipped)
-        )
-    if failed_symbols:
-        if stop_on_failure:
-            print_and_log("(not all tests are accounted for due to --stop-on-failure)")
-        print_and_log("Failed:")
-        for part, chapter, section in sorted(failed_symbols):
-            print_and_log("  - %s in %s / %s" % (section, part, chapter))
-
-    if generate_output and (failed == 0 or doc_even_if_error):
-        save_doctest_data(output_data)
-        return True
-
-    if failed == 0:
-        print("\nOK")
-    else:
-        print("\nFAILED")
-        return sys.exit(1)  # Travis-CI knows the tests have failed
-
 
 def load_doc_data():
     doc_html_data_path = get_doctest_html_data_path(should_be_readable=True)
@@ -437,36 +67,9 @@ def save_doctest_data(output_data):
         pickle.dump(output_data, output_file, 4)
 
 
-def write_doctest_data(quiet=False, reload=False):
-    """
-    Write internal (pickled) doc files and example data in docstrings.
-    """
-    if documentation is None:
-        print_and_log("documentation is not loaded.")
-        return
-
-    if not quiet:
-        print(f"Extracting internal doc data for {version_string}")
-        print("This may take a while...")
-
-    try:
-        output_data = load_doc_data() if reload else {}
-        for tests in documentation.get_tests():
-            create_output(tests, output_data)
-    except KeyboardInterrupt:
-        print("\nAborted.\n")
-        return
-
-    print("done.\n")
-    save_doctest_data(output_data)
-
-
 def main():
-    global check_partial_enlapsed_time
-    global definitions
-
     import_and_load_builtins()
-    definitions = Definitions(add_builtin=True)
+    md.DEFINITIONS = Definitions(add_builtin=True)
 
     parser = ArgumentParser(description="Mathics test suite.", add_help=False)
     parser.add_argument(
@@ -497,7 +100,7 @@ def main():
         default="",
         dest="exclude",
         metavar="SECTION",
-        help="excude SECTION(s). "
+        help="exclude SECTION(s). "
         "You can list multiple sections by adding a comma (and no space) in between section names.",
     )
     parser.add_argument(
@@ -519,7 +122,7 @@ def main():
     parser.add_argument(
         "--time-each",
         "-d",
-        dest="enlapsed_times",
+        dest="elapsed_times",
         action="store_true",
         help="check the time that take each test to parse, evaluate and compare.",
     )
@@ -573,39 +176,29 @@ def main():
         default=MAX_TESTS,
         help="run only  N tests",
     )
-    # FIXME: there is some weird interacting going on with
-    # mathics when tests in sorted order. Some of the Plot
-    # show a noticeable 2 minute delay in processing.
-    # I think the problem is in Mathics itself rather than
-    # sorting, but until we figure that out, use
-    # sort as an option only. For normal testing we don't
-    # want it for speed. But for document building which is
-    # rarely done, we do want sorting of the sections and chapters.
     parser.add_argument(
-        "--want-sorting",
-        dest="want_sorting",
+        "--show-statistics",
         action="store_true",
-        help="Sort chapters and sections",
+        help="print cache statistics",
     )
-    global logfile
+    global LOGFILE
 
     args = parser.parse_args()
 
-    if args.enlapsed_times:
-        check_partial_enlapsed_time = True
+    if args.elapsed_times:
+        md.CHECK_PARTIAL_ELAPSED_TIME = True
     # If a test for a specific section is called
     # just test it
     if args.logfilename:
-        logfile = open(args.logfilename, "wt")
+        md.LOGFILE = open(args.logfilename, "wt")
 
-    global documentation
-    documentation = MathicsDjangoDocumentation()
+    md.DOCUMENTATION = DjangoDocumentation()
 
     # LoadModule Mathics3 modules
     if args.pymathics:
         for module_name in args.pymathics.split(","):
             try:
-                eval_LoadModule(module_name, definitions)
+                eval_LoadModule(module_name, md.DEFINITIONS)
             except PyMathicsLoadException:
                 print(f"Python module {module_name} is not a Mathics3 module.")
 
@@ -614,11 +207,15 @@ def main():
             else:
                 print(f"Mathics3 Module {module_name} loaded")
 
-    documentation.gather_doctest_data()
+    md.DOCUMENTATION.gather_doctest_data()
+
+    start_time = None
+    total = 0
 
     if args.sections:
         sections = set(args.sections.split(","))
 
+        start_time = datetime.now()
         test_sections(
             sections,
             stop_on_failure=args.stop_on_failure,
@@ -626,9 +223,10 @@ def main():
             reload=args.reload,
         )
     elif args.chapters:
+        start_time = datetime.now()
         chapters = set(args.chapters.split(","))
 
-        test_chapters(
+        total = test_chapters(
             chapters, stop_on_failure=args.stop_on_failure, reload=args.reload
         )
     else:
@@ -636,13 +234,12 @@ def main():
             write_doctest_data(
                 quiet=args.quiet,
                 reload=args.reload,
-                want_sorting=args.want_sorting,
             )
         else:
             excludes = set(args.exclude.split(","))
             start_at = args.skip + 1
             start_time = datetime.now()
-            test_all(
+            total = test_all(
                 quiet=args.quiet,
                 generate_output=args.output,
                 stop_on_failure=args.stop_on_failure,
@@ -650,12 +247,16 @@ def main():
                 count=args.count,
                 doc_even_if_error=args.keep_going,
                 excludes=excludes,
-                want_sorting=args.want_sorting,
             )
             end_time = datetime.now()
             print("Tests took ", end_time - start_time)
-    if logfile:
-        logfile.close()
+
+    if total > 0 and start_time is not None:
+        end_time = datetime.now()
+        print("Test evalation took ", end_time - start_time)
+
+    if md.LOGFILE:
+        md.LOGFILE.close()
 
 
 if __name__ == "__main__":
diff --git a/mathics_django/settings.py b/mathics_django/settings.py
index 96fa92fd8..d618d8013 100644
--- a/mathics_django/settings.py
+++ b/mathics_django/settings.py
@@ -74,7 +74,7 @@ def get_bool_from_environment(env_var: str, default_value: str):
 )
 
 # We need another version as a fallback, and that is distributed with the
-# package. It is note user writable and not in the user space.
+# package. It is not user writable and not in the user space.
 DOC_SYSTEM_HTML_DATA_PATH = os.environ.get(
     "DOC_SYSTEM_HTML_DATA_PATH", osp.join(ROOT_DIR, "doc", "doc_html_data.pcl")
 )
diff --git a/mathics_django/web/controllers/doc.py b/mathics_django/web/controllers/doc.py
index 191be91f6..5e7bbcb58 100644
--- a/mathics_django/web/controllers/doc.py
+++ b/mathics_django/web/controllers/doc.py
@@ -8,25 +8,48 @@
 from django.core.handlers.wsgi import WSGIRequest
 from django.http import Http404, HttpResponse
 from django.shortcuts import render
-from mathics.eval.pymathics import pymathics_modules
+from mathics.doc.common_doc import get_module_doc, mathics3_module_part
+from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules
 
 from mathics_django.doc import documentation
 from mathics_django.doc.django_doc import (
     DjangoDocChapter,
     DjangoDocPart,
     DjangoDocSection,
-    MathicsDjangoDocumentation,
 )
 from mathics_django.web.views import JsonResponse
 
 DocResponse = Union[HttpResponse, JsonResponse]
 
+seen_pymathics_modules = copy(pymathics_modules)
+
 
 def check_for_pymathics_load():
+    global seen_pymathics_modules
     if seen_pymathics_modules != pymathics_modules:
-        # print("XXX refresh pymathics doc")
-        global documentation
-        documentation = MathicsDjangoDocumentation()
+        print("XXX refresh pymathics doc", pymathics_modules)
+        new_modules = pymathics_modules - seen_pymathics_modules
+        for new_module in new_modules:
+            title, _ = get_module_doc(new_module)
+            chapter = mathics3_module_part.doc.gather_chapter_doc_fn(
+                mathics3_module_part,
+                title,
+                mathics3_module_part.doc,
+            )
+            from trepan.api import debug
+
+            debug()
+            submodule_names_seen = set()
+            chapter.doc.doc_chapter(
+                new_module,
+                mathics3_module_part,
+                pymathics_builtins_by_module,
+                seen_pymathics_modules,
+                submodule_names_seen,
+            )
+            chapter.get_tests()
+        seen_pymathics_modules = copy(pymathics_modules)
+        pass
 
 
 def doc(request: WSGIRequest, ajax: bool = False) -> DocResponse:
@@ -44,7 +67,9 @@ def doc(request: WSGIRequest, ajax: bool = False) -> DocResponse:
 
 def doc_chapter(request: WSGIRequest, part, chapter, ajax: bool = False) -> DocResponse:
     """
-    Produces HTML via jinja templating for a chapter. Some examples of Chapters:
+    Produces HTML via jinja templating for a chapter. Some examples of
+    Chapters:
+
     * Introduction (in part Manual)
     * Procedural Programming (in part Reference of Built-in Symbols)
     """
@@ -87,9 +112,6 @@ def doc_part(request: WSGIRequest, part, ajax: bool = False) -> DocResponse:
     )
 
 
-seen_pymathics_modules = copy(pymathics_modules)
-
-
 def doc_search(request: WSGIRequest) -> DocResponse:
     check_for_pymathics_load()
     query = request.GET.get("query", "")

From 9182be19e93ec1d492162b3f9efdc424cc21dc9b Mon Sep 17 00:00:00 2001
From: rocky <rb@dustyfeet.com>
Date: Fri, 9 Feb 2024 03:12:17 -0500
Subject: [PATCH 03/23] Get Mathics3 Modules to loading properly ...

This needs the doc-code-rebased-rebased branch of Mathics core.
---
 .github/workflows/osx.yaml            |  2 +-
 .github/workflows/ubuntu.yml          |  2 +-
 .github/workflows/windows.yml         |  2 +-
 mathics_django/doc/django_doc.py      |  6 +--
 mathics_django/docpipeline.py         | 20 ++-------
 mathics_django/web/controllers/doc.py | 64 ++++++++++++++-------------
 6 files changed, 42 insertions(+), 54 deletions(-)

diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml
index e8c7af1ff..6257100bb 100644
--- a/.github/workflows/osx.yaml
+++ b/.github/workflows/osx.yaml
@@ -30,7 +30,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone https://github.com/Mathics3/mathics-core
+        git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index 11a13af08..a38b17ab6 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -29,7 +29,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone https://github.com/Mathics3/mathics-core
+        git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 1987c8612..1aa05578c 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -32,7 +32,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone https://github.com/Mathics3/mathics-core
+        git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core
         bash -c '(cd mathics-core && pip3 install -e .[full])'
         bash -c '(cd mathics-core && bash ./admin-tools/make-op-tables.sh)'
     - name: Install Mathics3 Django
diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index 392013669..fac8d0e28 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -18,7 +18,7 @@
     DocTests,
     DocText,
     Documentation,
-    XMLDoc,
+    DocumentationEntry,
     gather_tests,
     get_results_by_test,
     sorted_chapters,
@@ -80,7 +80,7 @@ def __init__(self):
         self.section_class = DjangoDocSection
         self.subsection_class = DjangoDocSubsection
 
-        self.gather_doctest_data()
+        self.load_documentation_sources()
         self.doctest_latex_pcl_path = settings.DOCTEST_LATEX_DATA_PCL
         self.pymathics_doc_loaded = False
         self.doc_data_file = settings.get_doctest_latex_data_path(
@@ -148,7 +148,7 @@ def search_sections(section, result):
         return sorted_results
 
 
-class DjangoDoc(XMLDoc):
+class DjangoDoc(DocumentationEntry):
     def __init__(self, doc, title, section, key_prefix=None):
         self.title = title
         if key_prefix is None:
diff --git a/mathics_django/docpipeline.py b/mathics_django/docpipeline.py
index 7d0b96418..cfc0d5d23 100644
--- a/mathics_django/docpipeline.py
+++ b/mathics_django/docpipeline.py
@@ -17,16 +17,10 @@
 import mathics
 import mathics.docpipeline as md
 from mathics.core.definitions import Definitions
-from mathics.core.load_builtin import (
-    builtins_by_module,
-    builtins_dict,
-    import_and_load_builtins,
-)
+from mathics.core.load_builtin import import_and_load_builtins
+from mathics.doc.utils import open_ensure_dir
 from mathics.docpipeline import (
     MAX_TESTS,
-    create_output,
-    open_ensure_dir,
-    print_and_log,
     test_all,
     test_chapters,
     test_sections,
@@ -38,13 +32,6 @@
 from mathics_django.settings import get_doctest_html_data_path
 
 
-def load_doc_data():
-    doc_html_data_path = get_doctest_html_data_path(should_be_readable=True)
-    print(f"Loading internal document data from {doc_html_data_path}")
-    with open_ensure_dir(doc_html_data_path, "rb") as doc_data_file:
-        return pickle.load(doc_data_file)
-
-
 def save_doctest_data(output_data):
     """
     Save doctest tests and test results to a Python PCL file.
@@ -207,7 +194,7 @@ def main():
             else:
                 print(f"Mathics3 Module {module_name} loaded")
 
-    md.DOCUMENTATION.gather_doctest_data()
+    # md.DOCUMENTATION.load_documentation_sources()
 
     start_time = None
     total = 0
@@ -244,7 +231,6 @@ def main():
                 generate_output=args.output,
                 stop_on_failure=args.stop_on_failure,
                 start_at=start_at,
-                count=args.count,
                 doc_even_if_error=args.keep_going,
                 excludes=excludes,
             )
diff --git a/mathics_django/web/controllers/doc.py b/mathics_django/web/controllers/doc.py
index 5e7bbcb58..9c10e4d8e 100644
--- a/mathics_django/web/controllers/doc.py
+++ b/mathics_django/web/controllers/doc.py
@@ -8,7 +8,8 @@
 from django.core.handlers.wsgi import WSGIRequest
 from django.http import Http404, HttpResponse
 from django.shortcuts import render
-from mathics.doc.common_doc import get_module_doc, mathics3_module_part
+from mathics.doc.common_doc import MATHICS3_MODULES_TITLE
+from mathics.doc.utils import slugify
 from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules
 
 from mathics_django.doc import documentation
@@ -23,37 +24,38 @@
 
 seen_pymathics_modules = copy(pymathics_modules)
 
+MATHICS3_MODULES_SLUG = slugify(MATHICS3_MODULES_TITLE)
 
-def check_for_pymathics_load():
+
+def check_for_new_load_modules():
+    """
+    See if we have laoded any new Mathics3 modules since the last time
+    we checked. If so get an add the documenation for that.
+    """
     global seen_pymathics_modules
     if seen_pymathics_modules != pymathics_modules:
-        print("XXX refresh pymathics doc", pymathics_modules)
-        new_modules = pymathics_modules - seen_pymathics_modules
-        for new_module in new_modules:
-            title, _ = get_module_doc(new_module)
-            chapter = mathics3_module_part.doc.gather_chapter_doc_fn(
-                mathics3_module_part,
-                title,
-                mathics3_module_part.doc,
-            )
-            from trepan.api import debug
-
-            debug()
-            submodule_names_seen = set()
-            chapter.doc.doc_chapter(
-                new_module,
-                mathics3_module_part,
-                pymathics_builtins_by_module,
-                seen_pymathics_modules,
-                submodule_names_seen,
-            )
-            chapter.get_tests()
+        mathics3_module_part = documentation.parts_by_slug.get(
+            MATHICS3_MODULES_SLUG, None
+        )
+        if mathics3_module_part is None:
+            print("Something is wrong: mathics3_module variable should not be None")
+            return
+        else:
+            # The "Mathics3 modules" part already exists; add the new chapters.
+            new_modules = pymathics_modules - seen_pymathics_modules
+            for new_module in new_modules:
+                chapter = documentation.doc_chapter(
+                    new_module, mathics3_module_part, pymathics_builtins_by_module
+                )
+                mathics3_module_part.chapters.append(chapter)
+                pass
+            pass
         seen_pymathics_modules = copy(pymathics_modules)
-        pass
+    return
 
 
 def doc(request: WSGIRequest, ajax: bool = False) -> DocResponse:
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     return render_doc(
         request,
         "overview.html",
@@ -73,7 +75,7 @@ def doc_chapter(request: WSGIRequest, part, chapter, ajax: bool = False) -> DocR
     * Introduction (in part Manual)
     * Procedural Programming (in part Reference of Built-in Symbols)
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     chapter = documentation.get_chapter(part, chapter)
     if not chapter:
         raise Http404
@@ -96,7 +98,7 @@ def doc_part(request: WSGIRequest, part, ajax: bool = False) -> DocResponse:
     * Manual
     * Reference of Built-in Symbols
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     part = documentation.get_part(part)
     if not part:
         raise Http404
@@ -113,7 +115,7 @@ def doc_part(request: WSGIRequest, part, ajax: bool = False) -> DocResponse:
 
 
 def doc_search(request: WSGIRequest) -> DocResponse:
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     query = request.GET.get("query", "")
     result = documentation.search(query)
     if len([item for exact, item in result if exact]) <= 1:
@@ -170,7 +172,7 @@ def doc_section(
     * A list of builtin-functions under a Guide Section. For example: Color Directives.
       The guide section here would be Colors.
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     section_obj = documentation.get_section(part, chapter, section)
     if not section_obj:
         raise Http404
@@ -204,7 +206,7 @@ def doc_subsection(
     organized in a guide section are tagged as a section rather than a
     subsection.)
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     subsection_obj = documentation.get_subsection(part, chapter, section, subsection)
     if not subsection_obj:
         raise Http404
@@ -240,7 +242,7 @@ def render_doc(
     If ``ajax`` is True the should the ajax URI prefix, e.g. " it we pass the result
 
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     object = context.get("object")
     context.update(
         {

From c26e0d0498ad4664b99c45e780b38aa3aff81979 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Mon, 18 Mar 2024 10:10:15 -0300
Subject: [PATCH 04/23] improving_guide_handling

---
 mathics_django/doc/django_doc.py              | 6 +++---
 mathics_django/doc/utils.py                   | 6 ++++++
 mathics_django/web/templates/doc/chapter.html | 2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index fac8d0e28..112c33f26 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -211,7 +211,7 @@ def __init__(self, doc, title, is_reference=False):
 
     def get_collection(self):
         """Return a list of parts in this doc"""
-        return self.doc.parts
+        return self.documentation.parts
 
     def html(self):
         if len(self.tests) == 0:
@@ -268,7 +268,7 @@ def __str__(self):
     def get_collection(self):
         """Return a list of subsections for this section that this section belongs
         to."""
-        return self.chapter.sections
+        return self.chapter.all_sections
 
     def html_data(self):
         indices = set()
@@ -321,7 +321,7 @@ def __init__(
 
     def get_uri(self) -> str:
         """Return the URI of this section."""
-        return f"/{self.chapter.part.slug}/{self.chapter.slug}/guide/"
+        return f"/{self.chapter.part.slug}/{self.chapter.slug}/{self.slug}"
 
 
 class DjangoDocSubsection(DocSubsection, DjangoDocElement):
diff --git a/mathics_django/doc/utils.py b/mathics_django/doc/utils.py
index 9770f1c5c..4651f0e7f 100644
--- a/mathics_django/doc/utils.py
+++ b/mathics_django/doc/utils.py
@@ -141,6 +141,12 @@ def repl_hypertext(match):
                     text = None
                 if text is None:
                     text = content
+
+                # If the reference points to the documentation,
+                # modify the url...
+                if content.startswith("/doc/"):
+                    content = content[4:]
+                    content = f"javascript:loadDoc('{content}')"
                 return r'<a href="%s">%s</a>' % (content, text)
 
         text = HYPERTEXT_RE.sub(repl_hypertext, text)
diff --git a/mathics_django/web/templates/doc/chapter.html b/mathics_django/web/templates/doc/chapter.html
index 3fca17fdc..6305faaad 100644
--- a/mathics_django/web/templates/doc/chapter.html
+++ b/mathics_django/web/templates/doc/chapter.html
@@ -11,7 +11,7 @@
 	{{ chapter.doc.html }}
 
 	<ul>
-		{% for section in chapter.sections %}
+		{% for section in chapter.all_sections %}
 		  {% if section.summary_text %}
             <li>{{ section|link:ajax }} &mdash; <i>{{section.summary_text}}</i></li>
           {% else %}

From 69b151a76d00033bc974a499c6bcdfb88f8ed323 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Tue, 19 Mar 2024 10:04:53 -0300
Subject: [PATCH 05/23] last fixes

---
 .github/workflows/osx.yaml                    |  2 +-
 .github/workflows/ubuntu.yml                  |  2 +-
 .github/workflows/windows.yml                 |  2 +-
 mathics_django/doc/django_doc.py              | 21 ++++++++++++-------
 mathics_django/web/templates/doc/chapter.html |  2 +-
 5 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml
index 6257100bb..3d60387de 100644
--- a/.github/workflows/osx.yaml
+++ b/.github/workflows/osx.yaml
@@ -30,7 +30,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core
+        git clone --branch more_docpipeline_fixes_2 https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index a38b17ab6..e3c222a26 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -29,7 +29,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core
+        git clone --branch more_docpipeline_fixes_2 https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 1aa05578c..70a84b973 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -32,7 +32,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core
+        git clone --branch more_docpipeline_fixes_2 https://github.com/Mathics3/mathics-core
         bash -c '(cd mathics-core && pip3 install -e .[full])'
         bash -c '(cd mathics-core && bash ./admin-tools/make-op-tables.sh)'
     - name: Install Mathics3 Django
diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index 112c33f26..fb4c6eb65 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -72,13 +72,7 @@ def get_title_html(self):
 class DjangoDocumentation(Documentation, DjangoDocElement):
     def __init__(self):
         super(DjangoDocumentation, self).__init__()
-        self.doc_class = DjangoDoc
         self.doc_dir = settings.DOC_DIR
-        self.chapter_class = DjangoDocChapter
-        self.guide_section_class = DjangoDocGuideSection
-        self.part_class = DjangoDocPart
-        self.section_class = DjangoDocSection
-        self.subsection_class = DjangoDocSubsection
 
         self.load_documentation_sources()
         self.doctest_latex_pcl_path = settings.DOCTEST_LATEX_DATA_PCL
@@ -88,6 +82,14 @@ def __init__(self):
         )
         self.title = "Overview"
 
+    def _set_classes(self):
+        self.doc_class = DjangoDoc
+        self.chapter_class = DjangoDocChapter
+        self.guide_section_class = DjangoDocGuideSection
+        self.part_class = DjangoDocPart
+        self.section_class = DjangoDocSection
+        self.subsection_class = DjangoDocSubsection
+        
     def __str__(self):
         return "\n\n\n".join(str(part) for part in self.parts)
 
@@ -195,7 +197,12 @@ class DjangoDocChapter(DocChapter, DjangoDocElement):
     """An object for a Django Documentation Chapter.
     A Chapter is part of a Part and contains Sections.
     """
-
+    @property
+    def guide_or_symbol_sections(self):
+        if self.guide_sections:
+            return self.guide_sections
+        return self.sections
+    
     def get_collection(self):
         """Return a list of chapters in the part of this chapter."""
         return self.part.chapters
diff --git a/mathics_django/web/templates/doc/chapter.html b/mathics_django/web/templates/doc/chapter.html
index 6305faaad..85363a84f 100644
--- a/mathics_django/web/templates/doc/chapter.html
+++ b/mathics_django/web/templates/doc/chapter.html
@@ -11,7 +11,7 @@
 	{{ chapter.doc.html }}
 
 	<ul>
-		{% for section in chapter.all_sections %}
+		{% for section in chapter.guide_or_symbol_sections %}
 		  {% if section.summary_text %}
             <li>{{ section|link:ajax }} &mdash; <i>{{section.summary_text}}</i></li>
           {% else %}

From 6c214004317a8bc5c4de78c976d643eefad5eef7 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Tue, 19 Mar 2024 10:07:57 -0300
Subject: [PATCH 06/23] black

---
 mathics_django/doc/django_doc.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index fb4c6eb65..aac8701c0 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -89,7 +89,7 @@ def _set_classes(self):
         self.part_class = DjangoDocPart
         self.section_class = DjangoDocSection
         self.subsection_class = DjangoDocSubsection
-        
+
     def __str__(self):
         return "\n\n\n".join(str(part) for part in self.parts)
 
@@ -197,12 +197,13 @@ class DjangoDocChapter(DocChapter, DjangoDocElement):
     """An object for a Django Documentation Chapter.
     A Chapter is part of a Part and contains Sections.
     """
+
     @property
     def guide_or_symbol_sections(self):
         if self.guide_sections:
             return self.guide_sections
         return self.sections
-    
+
     def get_collection(self):
         """Return a list of chapters in the part of this chapter."""
         return self.part.chapters

From b7202d2864a7ccf9a0a73d00201c28a76178f326 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Tue, 19 Mar 2024 10:46:56 -0300
Subject: [PATCH 07/23] fixes

---
 mathics_django/doc/django_doc.py | 152 ++-----------------------------
 1 file changed, 9 insertions(+), 143 deletions(-)

diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index aac8701c0..a593357c6 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -151,23 +151,16 @@ def search_sections(section, result):
 
 
 class DjangoDoc(DocumentationEntry):
-    def __init__(self, doc, title, section, key_prefix=None):
-        self.title = title
-        if key_prefix is None:
-            if section is not None:
-                chapter = section.chapter
-                part = chapter.part
-                # Note: we elide section.title
-                key_prefix = (part.title, chapter.title, title)
-            else:
-                key_prefix = None
-
-        self.rawdoc = doc
-        self.items = gather_tests(
-            self.rawdoc, DjangoDocTests, DjangoDocTest, DjangoDocText, key_prefix
-        )
-        return
+    def _set_classes(self):
+        """
+        Tells to the initializator the classes to be used to build the items.
+        This must be overloaded by the daughter classes.
+        """
+        self.docTest_collection_class = DjangoDocTests
+        self.docTest_class = DjangoDocTest
+        self.docText_class = DjangoDocText
 
+    
     def __str__(self):
         return "\n".join(str(item) for item in self.items)
 
@@ -238,38 +231,6 @@ class DjangoDocSection(DocSection, DjangoDocElement):
     """An object for a Django Documented Section.
     A Section is part of a Chapter. It can contain subsections.
     """
-
-    def __init__(
-        self,
-        chapter,
-        title: str,
-        text: str,
-        operator,
-        installed=True,
-        in_guide=False,
-        summary_text="",
-    ):
-        self.chapter = chapter
-        self.in_guide = in_guide
-        self.installed = installed
-        self.operator = operator
-        self.slug = slugify(title)
-        self.subsections = []
-        self.subsections_by_slug = {}
-        self.summary_text = summary_text
-        self.title = title
-
-        if text.count("<dl>") != text.count("</dl>"):
-            raise ValueError(
-                f"Missing opening or closing <dl> tag in {title} documentation"
-            )
-
-        # Needs to come after self.chapter is initialized since
-        # XMLDoc uses self.chapter.
-        self.doc = DjangoDoc(text, title, self)
-
-        chapter.sections_by_slug[self.slug] = self
-
     def __str__(self):
         return f"== {self.title} ==\n{self.doc}"
 
@@ -301,31 +262,6 @@ class DjangoDocGuideSection(DjangoDocSection, DjangoDocElement):
     like NamedColors or Orthogonal Polynomials.
     """
 
-    def __init__(
-        self, chapter: str, title: str, text: str, submodule, installed: bool = True
-    ):
-        self.chapter = chapter
-        self.doc = DjangoDoc(text, title, None)
-        self.in_guide = False
-        self.installed = installed
-        self.slug = slugify(title)
-        self.section = submodule
-        self.slug = slugify(title)
-        self.subsections = []
-        self.subsections_by_slug = {}
-        self.title = title
-
-        # FIXME: Sections never are operators. Subsections can have
-        # operators though.  Fix up the view and searching code not to
-        # look for the operator field of a section.
-        self.operator = False
-
-        if text.count("<dl>") != text.count("</dl>"):
-            raise ValueError(
-                f"Missing opening or closing <dl> tag in {title} documentation"
-            )
-        # print("YYY Adding section", title)
-        chapter.sections_by_slug[self.slug] = self
 
     def get_uri(self) -> str:
         """Return the URI of this section."""
@@ -336,76 +272,6 @@ class DjangoDocSubsection(DocSubsection, DjangoDocElement):
     """An object for a Django Documented Subsection.
     A Subsection is part of a Section.
     """
-
-    def __init__(
-        self,
-        chapter,
-        section,
-        title,
-        text,
-        operator=None,
-        installed=True,
-        in_guide=False,
-        summary_text="",
-    ):
-        """
-        Information that goes into a subsection object. This can be a written text, or
-        text extracted from the docstring of a builtin module or class.
-
-        About some of the parameters...
-
-        Some built-in classes are Operators. These are documented in a
-        slightly special way.
-
-        Some built-in require special libraries. When those libraries are not available,
-        parameter "installed" is False.
-
-        Some of the subsections are contained in a grouping module and need special work to
-        get the grouping module name correct.
-
-        For example the Chapter "Colors" is a module so the docstring
-        text for it is in mathics/builtin/colors/__init__.py . In
-        mathics/builtin/colors/named-colors.py we have the "section"
-        name for the class Read (the subsection) inside it.
-        """
-
-        title_summary_text = re.split(" -- ", title)
-        n = len(title_summary_text)
-        self.title = title_summary_text[0] if n > 0 else ""
-        self.summary_text = title_summary_text[1] if n > 1 else summary_text
-
-        self.doc = DjangoDoc(text, title, section)
-        self.chapter = chapter
-        self.installed = installed
-        self.operator = operator
-
-        self.section = section
-        self.slug = slugify(title)
-        self.title = title
-
-        if section:
-            chapter = section.chapter
-            part = chapter.part
-            # Note: we elide section.title
-            key_prefix = (part.title, chapter.title, title)
-        else:
-            key_prefix = None
-
-        if in_guide:
-            # Tests haven't been picked out yet from the doc string yet.
-            # Gather them here.
-            self.items = gather_tests(
-                text, DjangoDocTests, DjangoDocTest, DjangoDocText, key_prefix
-            )
-        else:
-            self.items = []
-
-        if text.count("<dl>") != text.count("</dl>"):
-            raise ValueError(
-                f"Missing opening or closing <dl> tag in {title} documentation"
-            )
-        self.section.subsections_by_slug[self.slug] = self
-
     def __str__(self):
         return f"=== {self.title} ===\n{self.doc}"
 

From 7219809d424d6e0939dabdc34a0a290ffa2f9670 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Tue, 19 Mar 2024 10:53:16 -0300
Subject: [PATCH 08/23] black

---
 mathics_django/doc/django_doc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index a593357c6..caa0a2851 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -160,7 +160,6 @@ def _set_classes(self):
         self.docTest_class = DjangoDocTest
         self.docText_class = DjangoDocText
 
-    
     def __str__(self):
         return "\n".join(str(item) for item in self.items)
 
@@ -231,6 +230,7 @@ class DjangoDocSection(DocSection, DjangoDocElement):
     """An object for a Django Documented Section.
     A Section is part of a Chapter. It can contain subsections.
     """
+
     def __str__(self):
         return f"== {self.title} ==\n{self.doc}"
 
@@ -262,7 +262,6 @@ class DjangoDocGuideSection(DjangoDocSection, DjangoDocElement):
     like NamedColors or Orthogonal Polynomials.
     """
 
-
     def get_uri(self) -> str:
         """Return the URI of this section."""
         return f"/{self.chapter.part.slug}/{self.chapter.slug}/{self.slug}"
@@ -272,6 +271,7 @@ class DjangoDocSubsection(DocSubsection, DjangoDocElement):
     """An object for a Django Documented Subsection.
     A Subsection is part of a Section.
     """
+
     def __str__(self):
         return f"=== {self.title} ===\n{self.doc}"
 

From 37362b7d05497d6375b5b92c4035be3e16f29a37 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Tue, 19 Mar 2024 11:06:04 -0300
Subject: [PATCH 09/23] windows workflow

---
 .github/workflows/windows.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 70a84b973..05e1afd57 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         os: [windows]
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.11']
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

From 93c1bbe3dc6be4ef353d2fc9c2c6f7ad482818c1 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Fri, 29 Mar 2024 13:21:19 -0300
Subject: [PATCH 10/23] init for DjangoDocGuideSection

---
 mathics_django/doc/django_doc.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index caa0a2851..7384935e6 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -261,6 +261,17 @@ class DjangoDocGuideSection(DjangoDocSection, DjangoDocElement):
     are examples of Guide Sections, and each contains a number of Sections.
     like NamedColors or Orthogonal Polynomials.
     """
+    def __init__(
+        self,
+        chapter: DocChapter,
+        title: str,
+        text: str,
+        submodule,
+        installed: bool = True,
+    ):
+        super().__init__(chapter, title, text, None, installed, False)
+        self.section = submodule
+
 
     def get_uri(self) -> str:
         """Return the URI of this section."""

From 59d067583a55f475bf58856854aa3b9e9161ae63 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Fri, 29 Mar 2024 13:28:47 -0300
Subject: [PATCH 11/23] black

---
 mathics_django/doc/django_doc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index 7384935e6..c54c41c91 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -261,6 +261,7 @@ class DjangoDocGuideSection(DjangoDocSection, DjangoDocElement):
     are examples of Guide Sections, and each contains a number of Sections.
     like NamedColors or Orthogonal Polynomials.
     """
+
     def __init__(
         self,
         chapter: DocChapter,
@@ -272,7 +273,6 @@ def __init__(
         super().__init__(chapter, title, text, None, installed, False)
         self.section = submodule
 
-
     def get_uri(self) -> str:
         """Return the URI of this section."""
         return f"/{self.chapter.part.slug}/{self.chapter.slug}/{self.slug}"

From 765a45a16baa308f5b7cb34c95a8919b5e7a4bfb Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Thu, 1 Aug 2024 10:17:35 -0300
Subject: [PATCH 12/23] documentation.doc_chapter ->
 mathics.doc.gather.doc_chapter

---
 mathics_django/web/controllers/doc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mathics_django/web/controllers/doc.py b/mathics_django/web/controllers/doc.py
index 9c10e4d8e..51b571d2d 100644
--- a/mathics_django/web/controllers/doc.py
+++ b/mathics_django/web/controllers/doc.py
@@ -9,6 +9,7 @@
 from django.http import Http404, HttpResponse
 from django.shortcuts import render
 from mathics.doc.common_doc import MATHICS3_MODULES_TITLE
+from mathics.doc.gather import doc_chapter as gather_doc_chapter
 from mathics.doc.utils import slugify
 from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules
 
@@ -44,7 +45,7 @@ def check_for_new_load_modules():
             # The "Mathics3 modules" part already exists; add the new chapters.
             new_modules = pymathics_modules - seen_pymathics_modules
             for new_module in new_modules:
-                chapter = documentation.doc_chapter(
+                chapter = gather_doc_chapter(
                     new_module, mathics3_module_part, pymathics_builtins_by_module
                 )
                 mathics3_module_part.chapters.append(chapter)

From 95a761cca1019ffc725e262ee7e72693c273ab17 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Thu, 1 Aug 2024 10:24:03 -0300
Subject: [PATCH 13/23] pointing now to master

---
 .github/workflows/osx.yaml    | 2 +-
 .github/workflows/ubuntu.yml  | 2 +-
 .github/workflows/windows.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml
index 3d60387de..f89036b4c 100644
--- a/.github/workflows/osx.yaml
+++ b/.github/workflows/osx.yaml
@@ -30,7 +30,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch more_docpipeline_fixes_2 https://github.com/Mathics3/mathics-core
+        git clone --branch master https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index e3c222a26..733c383e3 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -29,7 +29,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch more_docpipeline_fixes_2 https://github.com/Mathics3/mathics-core
+        git clone --branch master https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 05e1afd57..e9d9c4619 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -32,7 +32,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch more_docpipeline_fixes_2 https://github.com/Mathics3/mathics-core
+        git clone --branch master https://github.com/Mathics3/mathics-core
         bash -c '(cd mathics-core && pip3 install -e .[full])'
         bash -c '(cd mathics-core && bash ./admin-tools/make-op-tables.sh)'
     - name: Install Mathics3 Django

From cae2f2b999c9f1bf1c375fb8c35273a6aba05e88 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Thu, 1 Aug 2024 13:45:02 -0300
Subject: [PATCH 14/23] mathics-core.master docpipeline compatibility

---
 .github/workflows/osx.yaml    |   2 +-
 .github/workflows/ubuntu.yml  |   2 +-
 .github/workflows/windows.yml |   2 +-
 mathics_django/docpipeline.py | 206 +++++-----------------------------
 4 files changed, 34 insertions(+), 178 deletions(-)

diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml
index f89036b4c..0c5b590de 100644
--- a/.github/workflows/osx.yaml
+++ b/.github/workflows/osx.yaml
@@ -30,7 +30,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch master https://github.com/Mathics3/mathics-core
+        git clone --branch docpipeline_django_compat https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index 733c383e3..59a64efd8 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -29,7 +29,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch master https://github.com/Mathics3/mathics-core
+        git clone --branch docpipeline_django_compat https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index e9d9c4619..5f251b5ad 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -32,7 +32,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch master https://github.com/Mathics3/mathics-core
+        git clone --branch docpipeline_django_compat https://github.com/Mathics3/mathics-core
         bash -c '(cd mathics-core && pip3 install -e .[full])'
         bash -c '(cd mathics-core && bash ./admin-tools/make-op-tables.sh)'
     - name: Install Mathics3 Django
diff --git a/mathics_django/docpipeline.py b/mathics_django/docpipeline.py
index cfc0d5d23..e22cffbe8 100644
--- a/mathics_django/docpipeline.py
+++ b/mathics_django/docpipeline.py
@@ -20,7 +20,9 @@
 from mathics.core.load_builtin import import_and_load_builtins
 from mathics.doc.utils import open_ensure_dir
 from mathics.docpipeline import (
+    DocTestPipeline,
     MAX_TESTS,
+    build_arg_parser,
     test_all,
     test_chapters,
     test_sections,
@@ -55,194 +57,48 @@ def save_doctest_data(output_data):
 
 
 def main():
-    import_and_load_builtins()
-    md.DEFINITIONS = Definitions(add_builtin=True)
-
-    parser = ArgumentParser(description="Mathics test suite.", add_help=False)
-    parser.add_argument(
-        "--help", "-h", help="show this help message and exit", action="help"
-    )
-    parser.add_argument(
-        "--version", "-v", action="version", version="%(prog)s " + mathics.__version__
-    )
-    parser.add_argument(
-        "--chapters",
-        "-c",
-        dest="chapters",
-        metavar="CHAPTER",
-        help="only test CHAPTER(s). "
-        "You can list multiple chapters by adding a comma (and no space) in between chapter names.",
-    )
-    parser.add_argument(
-        "--sections",
-        "-s",
-        dest="sections",
-        metavar="SECTION",
-        help="only test SECTION(s). "
-        "You can list multiple sections by adding a comma (and no space) in between section names.",
-    )
-    parser.add_argument(
-        "--exclude",
-        "-X",
-        default="",
-        dest="exclude",
-        metavar="SECTION",
-        help="exclude SECTION(s). "
-        "You can list multiple sections by adding a comma (and no space) in between section names.",
-    )
-    parser.add_argument(
-        "--load-module",
-        "-l",
-        dest="pymathics",
-        metavar="MATHIC3-MODULES",
-        help="load Mathics3 module MATHICS3-MODULES. "
-        "You can list multiple Mathics3 Modules by adding a comma (and no space) in between "
-        "module names.",
-    )
-    parser.add_argument(
-        "--logfile",
-        "-f",
-        dest="logfilename",
-        metavar="LOGFILENAME",
-        help="stores the output in [logfilename]. ",
-    )
-    parser.add_argument(
-        "--time-each",
-        "-d",
-        dest="elapsed_times",
-        action="store_true",
-        help="check the time that take each test to parse, evaluate and compare.",
-    )
-
-    parser.add_argument(
-        "--output",
-        "-o",
-        dest="output",
-        action="store_true",
-        help="generate pickled internal document data",
-    )
-    parser.add_argument(
-        "--reload",
-        "-r",
-        dest="reload",
-        action="store_true",
-        help="reload pickled internal data, before possibly adding to it",
-    )
-    parser.add_argument(
-        "--doc-only",
-        dest="doc_only",
-        action="store_true",
-        help="reload pickled internal document data, before possibly adding to it",
-    )
-    parser.add_argument(
-        "--quiet", "-q", dest="quiet", action="store_true", help="hide passed tests"
-    )
-    parser.add_argument(
-        "--keep-going",
-        "-k",
-        dest="keep_going",
-        action="store_true",
-        help="create documentation even if there is a test failure",
-    )
-    parser.add_argument(
-        "--stop-on-failure", "-x", action="store_true", help="stop on failure"
-    )
-    parser.add_argument(
-        "--skip",
-        metavar="N",
-        dest="skip",
-        type=int,
-        default=0,
-        help="skip the first N tests",
-    )
-    parser.add_argument(
-        "--count",
-        metavar="N",
-        dest="count",
-        type=int,
-        default=MAX_TESTS,
-        help="run only  N tests",
-    )
-    parser.add_argument(
-        "--show-statistics",
-        action="store_true",
-        help="print cache statistics",
+    args = build_arg_parser()
+    data_path =  (
+        get_doctest_html_data_path(
+            should_be_readable=False, create_parent=True
+        )
+        if args.output
+        else None
     )
-    global LOGFILE
-
-    args = parser.parse_args()
-
-    if args.elapsed_times:
-        md.CHECK_PARTIAL_ELAPSED_TIME = True
-    # If a test for a specific section is called
-    # just test it
-    if args.logfilename:
-        md.LOGFILE = open(args.logfilename, "wt")
-
-    md.DOCUMENTATION = DjangoDocumentation()
-
-    # LoadModule Mathics3 modules
-    if args.pymathics:
-        for module_name in args.pymathics.split(","):
-            try:
-                eval_LoadModule(module_name, md.DEFINITIONS)
-            except PyMathicsLoadException:
-                print(f"Python module {module_name} is not a Mathics3 module.")
-
-            except ImportError:
-                print(f"Python module {module_name} does not exist")
-            else:
-                print(f"Mathics3 Module {module_name} loaded")
-
-    # md.DOCUMENTATION.load_documentation_sources()
-
-    start_time = None
-    total = 0
+    test_pipeline = DocTestPipeline(args, output_format="xml", data_path=data_path)
+    test_status = test_pipeline.status
 
     if args.sections:
-        sections = set(args.sections.split(","))
-
+        include_sections = set(args.sections.split(","))
+        exclude_subsections = set(args.exclude.split(","))
         start_time = datetime.now()
-        test_sections(
-            sections,
-            stop_on_failure=args.stop_on_failure,
-            generate_output=args.output,
-            reload=args.reload,
-        )
+        test_sections(test_pipeline, include_sections, exclude_subsections)
     elif args.chapters:
         start_time = datetime.now()
-        chapters = set(args.chapters.split(","))
-
-        total = test_chapters(
-            chapters, stop_on_failure=args.stop_on_failure, reload=args.reload
-        )
+        include_chapters = set(args.chapters.split(","))
+        exclude_sections = set(args.exclude.split(","))
+        test_chapters(test_pipeline, include_chapters, exclude_sections)
     else:
         if args.doc_only:
-            write_doctest_data(
-                quiet=args.quiet,
-                reload=args.reload,
-            )
+            write_doctest_data(test_pipeline)
         else:
             excludes = set(args.exclude.split(","))
-            start_at = args.skip + 1
             start_time = datetime.now()
-            total = test_all(
-                quiet=args.quiet,
-                generate_output=args.output,
-                stop_on_failure=args.stop_on_failure,
-                start_at=start_at,
-                doc_even_if_error=args.keep_going,
-                excludes=excludes,
-            )
-            end_time = datetime.now()
-            print("Tests took ", end_time - start_time)
+            test_all(test_pipeline, excludes=excludes)
 
-    if total > 0 and start_time is not None:
-        end_time = datetime.now()
-        print("Test evalation took ", end_time - start_time)
+    if test_status.total > 0 and start_time is not None:
+        print("Test evaluation took ", datetime.now() - start_time)
 
-    if md.LOGFILE:
-        md.LOGFILE.close()
+    if test_pipeline.logfile:
+        test_pipeline.logfile.close()
+    if args.show_statistics:
+        show_lru_cache_statistics()
+
+    if test_status.failed == 0:
+        print("\nOK")
+    else:
+        print("\nFAILED")
+        sys.exit(1)  # Travis-CI knows the tests have failed
 
 
 if __name__ == "__main__":

From 344f13e883e55157a04cc4cb8aaa3e276ea4ae3f Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Thu, 1 Aug 2024 13:49:41 -0300
Subject: [PATCH 15/23] black

---
 mathics_django/docpipeline.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/mathics_django/docpipeline.py b/mathics_django/docpipeline.py
index e22cffbe8..69b97c6a8 100644
--- a/mathics_django/docpipeline.py
+++ b/mathics_django/docpipeline.py
@@ -58,10 +58,8 @@ def save_doctest_data(output_data):
 
 def main():
     args = build_arg_parser()
-    data_path =  (
-        get_doctest_html_data_path(
-            should_be_readable=False, create_parent=True
-        )
+    data_path = (
+        get_doctest_html_data_path(should_be_readable=False, create_parent=True)
         if args.output
         else None
     )

From e01963e4456200c812501b3f497d6b142b7bf041 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Thu, 1 Aug 2024 13:54:01 -0300
Subject: [PATCH 16/23] increasing python version in workflow

---
 .github/workflows/osx.yaml   | 2 +-
 .github/workflows/ubuntu.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml
index 0c5b590de..406b218a5 100644
--- a/.github/workflows/osx.yaml
+++ b/.github/workflows/osx.yaml
@@ -11,7 +11,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.9', '3.10', '3.11']
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index 59a64efd8..686cc99b8 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

From 63acd00b057fd16fa469d458d7e3db51b8c099c4 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Thu, 1 Aug 2024 13:57:20 -0300
Subject: [PATCH 17/23] llvm in osx workflow

---
 .github/workflows/osx.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml
index 406b218a5..5e34d28f8 100644
--- a/.github/workflows/osx.yaml
+++ b/.github/workflows/osx.yaml
@@ -20,7 +20,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install OS dependencies
       run: |
-        brew install llvm@11 nodejs npm tesseract
+        brew install llvm nodejs npm tesseract
         python -m pip install --upgrade pip
         LLVM_CONFIG=/usr/local/Cellar/llvm@11/11.1.0/bin/llvm-config pip install llvmlite
     - name: Install Python dependencies

From 1a8f3f4ccdb1a7c53ea44b52ec88c356f82df140 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Thu, 1 Aug 2024 14:31:05 -0300
Subject: [PATCH 18/23] not load modules chapter twice

---
 mathics_django/web/controllers/doc.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/mathics_django/web/controllers/doc.py b/mathics_django/web/controllers/doc.py
index 51b571d2d..0aeb6e60a 100644
--- a/mathics_django/web/controllers/doc.py
+++ b/mathics_django/web/controllers/doc.py
@@ -40,17 +40,14 @@ def check_for_new_load_modules():
         )
         if mathics3_module_part is None:
             print("Something is wrong: mathics3_module variable should not be None")
-            return
-        else:
-            # The "Mathics3 modules" part already exists; add the new chapters.
-            new_modules = pymathics_modules - seen_pymathics_modules
-            for new_module in new_modules:
-                chapter = gather_doc_chapter(
-                    new_module, mathics3_module_part, pymathics_builtins_by_module
-                )
-                mathics3_module_part.chapters.append(chapter)
-                pass
-            pass
+
+        # The "Mathics3 modules" part already exists; add the new chapters.
+        new_modules = pymathics_modules - seen_pymathics_modules
+        for new_module in new_modules:
+            chapter = gather_doc_chapter(
+                new_module, mathics3_module_part, pymathics_builtins_by_module
+            )
+            #  mathics3_module_part.chapters.append(chapter)
         seen_pymathics_modules = copy(pymathics_modules)
     return
 

From 5c81ab61070e0954b1c9aae6f86c4a16cee91293 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Thu, 1 Aug 2024 14:40:24 -0300
Subject: [PATCH 19/23] cleaning unused libraries. Automatic reload docs when
 the window is open.

---
 mathics_django/docpipeline.py      | 11 ++---------
 mathics_django/web/media/js/doc.js |  3 ++-
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/mathics_django/docpipeline.py b/mathics_django/docpipeline.py
index 69b97c6a8..381d39559 100644
--- a/mathics_django/docpipeline.py
+++ b/mathics_django/docpipeline.py
@@ -9,28 +9,21 @@
    Mathics built-in functions
 2. Creates/updates internal documentation data
 """
-
 import pickle
-from argparse import ArgumentParser
+import sys
 from datetime import datetime
 
-import mathics
-import mathics.docpipeline as md
-from mathics.core.definitions import Definitions
-from mathics.core.load_builtin import import_and_load_builtins
 from mathics.doc.utils import open_ensure_dir
 from mathics.docpipeline import (
     DocTestPipeline,
-    MAX_TESTS,
     build_arg_parser,
     test_all,
     test_chapters,
     test_sections,
     write_doctest_data,
 )
-from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule
+from mathics.timing import show_lru_cache_statistics
 
-from mathics_django.doc import DjangoDocumentation
 from mathics_django.settings import get_doctest_html_data_path
 
 
diff --git a/mathics_django/web/media/js/doc.js b/mathics_django/web/media/js/doc.js
index 38e6cf791..0643f7979 100644
--- a/mathics_django/web/media/js/doc.js
+++ b/mathics_django/web/media/js/doc.js
@@ -60,7 +60,8 @@ function loadDoc(page) {
 }
 
 function showDoc() {
-	const docLink = document.getElementById('doclink');
+        const docLink = document.getElementById('doclink');
+        loadDoc('');
 
 	document.getElementById('doc').style.display = 'block';
 

From f0f18685ae993e69ac304fb8c0161af801993b88 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Fri, 2 Aug 2024 23:48:00 -0300
Subject: [PATCH 20/23] fix typos found by rocky and codespell.

---
 mathics_django/doc/django_doc.py        |  5 +----
 mathics_django/server.py                |  2 +-
 mathics_django/web/controllers/doc.py   |  6 +++---
 mathics_django/web/format.py            | 16 ++++++++--------
 mathics_django/web/templates/about.html |  4 ++--
 5 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index c54c41c91..f19657ae3 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -5,7 +5,6 @@
 """
 
 import pickle
-import re
 
 from django.utils.safestring import mark_safe
 from mathics import settings
@@ -19,11 +18,9 @@
     DocText,
     Documentation,
     DocumentationEntry,
-    gather_tests,
     get_results_by_test,
     sorted_chapters,
 )
-from mathics.doc.utils import slugify
 
 from mathics_django.doc.utils import escape_html
 from mathics_django.settings import get_doctest_html_data_path
@@ -176,7 +173,7 @@ def html(self):
             # In module-style docstring tagging, the first line of the docstring is the section title.
             # since that is tagged and shown as a title, it is redundant here is the section body.
             # Or that is the intent. This code is a bit hacky.
-            items[0].text = items[0].text[len(title_line) :]
+            items[0].text = items[0].text[len(title_line):]
 
         text = "\n".join(item.html() for item in items if not item.is_private())
         if text == "":
diff --git a/mathics_django/server.py b/mathics_django/server.py
index ca7527a52..4f9b5c916 100755
--- a/mathics_django/server.py
+++ b/mathics_django/server.py
@@ -31,7 +31,7 @@ def check_database():
     manage_file = osp.join(osp.dirname(osp.realpath(__file__)), "manage.py")
     try:
         subprocess.check_call([sys.executable, manage_file, "migrate", "--noinput"])
-        print("\ndatabase initialized sucessfully")
+        print("\ndatabase initialized successfully")
     except subprocess.CalledProcessError:
         print("error: failed to create database")
         sys.exit(1)
diff --git a/mathics_django/web/controllers/doc.py b/mathics_django/web/controllers/doc.py
index 0aeb6e60a..00b7a1394 100644
--- a/mathics_django/web/controllers/doc.py
+++ b/mathics_django/web/controllers/doc.py
@@ -30,8 +30,8 @@
 
 def check_for_new_load_modules():
     """
-    See if we have laoded any new Mathics3 modules since the last time
-    we checked. If so get an add the documenation for that.
+    See if we have loaded any new Mathics3 modules since the last time
+    we checked. If so get an add the documentation for that.
     """
     global seen_pymathics_modules
     if seen_pymathics_modules != pymathics_modules:
@@ -198,7 +198,7 @@ def doc_subsection(
     subsection: str,
     ajax: bool = False,
 ) -> DocResponse:
-    """Proceses a document subsection. This is often the bottom-most
+    """Processes a document subsection. This is often the bottom-most
     entity right now.  In particular it contains built-in functions
     which are part of a guide section.  (Those builtings that are not
     organized in a guide section are tagged as a section rather than a
diff --git a/mathics_django/web/format.py b/mathics_django/web/format.py
index ccf404eff..d5ce03e15 100644
--- a/mathics_django/web/format.py
+++ b/mathics_django/web/format.py
@@ -68,7 +68,7 @@ def eval_boxes(result, fn: Callable, obj, **options):
         return dict((k, obj.format_output(expr, f)) for k, f in format.items())
 
     # For some expressions, we want formatting to be different.
-    # In particular for FullForm output, we dont' want MathML, we want
+    # In particular for FullForm output, we don't want MathML, we want
     # plain-ol' text so we can cut and paste that.
 
     expr_type = expr.get_head_name()
@@ -142,7 +142,6 @@ def eval_boxes(result, fn: Callable, obj, **options):
 def hierarchy_pos(
     G, root=None, width=1.0, vert_gap=0.2, vert_loc=0, leaf_vs_root_factor=0.5
 ):
-
     """Position nodes in tree layout. The root is at the top.
 
     Based on Joel's answer at https://stackoverflow.com/a/29597209/2966723,
@@ -219,7 +218,7 @@ def hierarchy_pos(
         return cached_pair
 
     # These get swapped if tree edge directions point to the root.
-    decendants = nx.descendants
+    descendants = nx.descendants
     out_degree = G.out_degree if hasattr(G, "out_degree") else G.degree
     neighbors = G.neighbors
 
@@ -231,7 +230,7 @@ def hierarchy_pos(
                 # The case where we have a one or two node graph is ambiguous.
                 root = list(nx.topological_sort(G))[-1]
                 # Swap motion functions
-                decendants = nx.ancestors
+                descendants = nx.ancestors
                 out_degree = G.in_degree
                 neighbors = G.predecessors
             else:
@@ -306,7 +305,9 @@ def _hierarchy_pos(
 
     xcenter = width / 2.0
     if isinstance(G, nx.DiGraph):
-        leafcount = len([node for node in decendants(G, root) if out_degree(node) == 0])
+        leafcount = len(
+            [node for node in descendants(G, root) if out_degree(node) == 0]
+        )
     elif isinstance(G, nx.Graph):
         leafcount = len(
             [
@@ -397,7 +398,6 @@ def clamp(value, min=-math.inf, max=math.inf):
 
 
 def harmonize_parameters(G, draw_options: dict):
-
     global node_size
     graph_layout = G.graph_layout if hasattr(G, "graph_layout") else ""
 
@@ -480,7 +480,7 @@ def format_graph(G) -> str:
 
     # pyplot.tight_layout()
     svg_graph_xml = get_graph()
-    svg_str = svg_graph_xml[svg_graph_xml.find("<svg xmlns:xlink") :]
+    svg_str = svg_graph_xml[svg_graph_xml.find("<svg xmlns:xlink"):]
     return svg_str
 
 
@@ -497,5 +497,5 @@ def get_graph() -> str:
     buffer.close()
 
     # TODO: In the future if probably want to base64 encode.
-    # Use: base64.b64encode(some peice of image_svg)
+    # Use: base64.b64encode(some piece of image_svg)
     return graph_svg.decode("utf-8")
diff --git a/mathics_django/web/templates/about.html b/mathics_django/web/templates/about.html
index e390df2fc..2e492392c 100644
--- a/mathics_django/web/templates/about.html
+++ b/mathics_django/web/templates/about.html
@@ -139,10 +139,10 @@ <h1>Machine information</h1>
 				</ul>
 			</p>
 
-			<h1>Mathics3 Setttings</h1>
+			<h1>Mathics3 Settings</h1>
 			        <ul>
 					<li>Maximum Digits allowed in a String: {{MaximumDigitsInString}}</li>
-					<li>System Charecter Encoding: {{SystemCharacterEncoding}}</li>
+					<li>System Character Encoding: {{SystemCharacterEncoding}}</li>
 					<li>Time format is 12 Hour?: {{Time12Hour}}</li>
 			        </ul>
 

From 0801cdc27babb71965a8905e23fce596d8ccb0b7 Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Fri, 2 Aug 2024 23:52:45 -0300
Subject: [PATCH 21/23] redirecting mathics-core branch in workflows

---
 .github/workflows/osx.yaml    | 2 +-
 .github/workflows/ubuntu.yml  | 2 +-
 .github/workflows/windows.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml
index 5e34d28f8..14a54d02a 100644
--- a/.github/workflows/osx.yaml
+++ b/.github/workflows/osx.yaml
@@ -30,7 +30,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch docpipeline_django_compat https://github.com/Mathics3/mathics-core
+        git clone --branch main https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index 686cc99b8..d4bb8b483 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -29,7 +29,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch docpipeline_django_compat https://github.com/Mathics3/mathics-core
+        git clone --branch main https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index 5f251b5ad..bf61c121e 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -32,7 +32,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch docpipeline_django_compat https://github.com/Mathics3/mathics-core
+        git clone --branch main https://github.com/Mathics3/mathics-core
         bash -c '(cd mathics-core && pip3 install -e .[full])'
         bash -c '(cd mathics-core && bash ./admin-tools/make-op-tables.sh)'
     - name: Install Mathics3 Django

From 9e7098fa7bdfd96691413979b665a765c0f9a6ec Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Sat, 3 Aug 2024 12:50:18 -0300
Subject: [PATCH 22/23] workflows

---
 .github/workflows/osx.yaml    | 2 +-
 .github/workflows/ubuntu.yml  | 2 +-
 .github/workflows/windows.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml
index 14a54d02a..352b2b088 100644
--- a/.github/workflows/osx.yaml
+++ b/.github/workflows/osx.yaml
@@ -30,7 +30,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch main https://github.com/Mathics3/mathics-core
+        git clone --branch master https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index d4bb8b483..4b42bc7e4 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -29,7 +29,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch main https://github.com/Mathics3/mathics-core
+        git clone --branch master https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index bf61c121e..e9d9c4619 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -32,7 +32,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone --branch main https://github.com/Mathics3/mathics-core
+        git clone --branch master https://github.com/Mathics3/mathics-core
         bash -c '(cd mathics-core && pip3 install -e .[full])'
         bash -c '(cd mathics-core && bash ./admin-tools/make-op-tables.sh)'
     - name: Install Mathics3 Django

From ce785cd256f12bc6f4011f3ea594d082c6e03c4d Mon Sep 17 00:00:00 2001
From: mmatera <matera@fisica.unlp.edu.ar>
Date: Sat, 3 Aug 2024 15:18:53 -0300
Subject: [PATCH 23/23] adding comments. More docstrings

---
 mathics_django/doc/django_doc.py                       | 10 +++++++++-
 mathics_django/web/format.py                           |  2 +-
 mathics_django/web/migrations/0001_initial.py          |  1 -
 .../web/migrations/0002_auto_20200917_2354.py          |  1 -
 .../web/migrations/0003_auto_20200921_1835.py          |  1 -
 .../web/migrations/0004_auto_20210425_1408.py          |  1 -
 6 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py
index f19657ae3..dd5b42f28 100644
--- a/mathics_django/doc/django_doc.py
+++ b/mathics_django/doc/django_doc.py
@@ -41,6 +41,9 @@ class DjangoDocElement:
     """
 
     def href(self, ajax=False):
+        """Return the URI of the element. If the call is done
+        using ajax, the href is a javascript instruction for loading
+        the documentation from the corresponding uri."""
         if ajax:
             return f"javascript:loadDoc('{self.get_uri()}')"
         else:
@@ -63,6 +66,8 @@ def get_prev_next(self):
         return prev, next
 
     def get_title_html(self):
+        """Get the title of the element."""
+        # used by web/templatetags/doc.py
         return mark_safe(escape_html(self.title, single_line=True))
 
 
@@ -91,6 +96,7 @@ def __str__(self):
         return "\n\n\n".join(str(part) for part in self.parts)
 
     def get_uri(self) -> str:
+        """Return the URI of the documentation root."""
         return "/"
 
     def search(self, query):
@@ -173,7 +179,7 @@ def html(self):
             # In module-style docstring tagging, the first line of the docstring is the section title.
             # since that is tagged and shown as a title, it is redundant here is the section body.
             # Or that is the intent. This code is a bit hacky.
-            items[0].text = items[0].text[len(title_line):]
+            items[0].text = items[0].text[len(title_line) :]
 
         text = "\n".join(item.html() for item in items if not item.is_private())
         if text == "":
@@ -198,6 +204,7 @@ def get_collection(self):
         return self.part.chapters
 
     def get_uri(self) -> str:
+        """Return the URI of this chapter."""
         return f"/{self.part.slug}/{self.slug}/"
 
 
@@ -220,6 +227,7 @@ def html(self):
         )
 
     def get_uri(self) -> str:
+        """Return the URI of this part."""
         return f"/{self.slug}/"
 
 
diff --git a/mathics_django/web/format.py b/mathics_django/web/format.py
index d5ce03e15..0796bf796 100644
--- a/mathics_django/web/format.py
+++ b/mathics_django/web/format.py
@@ -480,7 +480,7 @@ def format_graph(G) -> str:
 
     # pyplot.tight_layout()
     svg_graph_xml = get_graph()
-    svg_str = svg_graph_xml[svg_graph_xml.find("<svg xmlns:xlink"):]
+    svg_str = svg_graph_xml[svg_graph_xml.find("<svg xmlns:xlink") :]
     return svg_str
 
 
diff --git a/mathics_django/web/migrations/0001_initial.py b/mathics_django/web/migrations/0001_initial.py
index de4a2b891..daaf823da 100644
--- a/mathics_django/web/migrations/0001_initial.py
+++ b/mathics_django/web/migrations/0001_initial.py
@@ -8,7 +8,6 @@
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = [
diff --git a/mathics_django/web/migrations/0002_auto_20200917_2354.py b/mathics_django/web/migrations/0002_auto_20200917_2354.py
index 5c4bb6536..f75505db2 100644
--- a/mathics_django/web/migrations/0002_auto_20200917_2354.py
+++ b/mathics_django/web/migrations/0002_auto_20200917_2354.py
@@ -6,7 +6,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("web", "0001_initial"),
     ]
diff --git a/mathics_django/web/migrations/0003_auto_20200921_1835.py b/mathics_django/web/migrations/0003_auto_20200921_1835.py
index 1aa529a5c..98641e525 100644
--- a/mathics_django/web/migrations/0003_auto_20200921_1835.py
+++ b/mathics_django/web/migrations/0003_auto_20200921_1835.py
@@ -8,7 +8,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("web", "0002_auto_20200917_2354"),
     ]
diff --git a/mathics_django/web/migrations/0004_auto_20210425_1408.py b/mathics_django/web/migrations/0004_auto_20210425_1408.py
index a95305619..29372a967 100644
--- a/mathics_django/web/migrations/0004_auto_20210425_1408.py
+++ b/mathics_django/web/migrations/0004_auto_20210425_1408.py
@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("web", "0003_auto_20200921_1835"),
     ]