[Automate CI/CD Workflow]

.github/workflows/publish_release.yml

@@ -0,0 +1,154 @@
+name: Publish the latest release on PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - release
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Override the version number'
+        required: false
+      rollback:
+        description: 'Trigger a rollback to the previous stable version'
+        required: false
+        default: false
+
+jobs:
+  validate-tag:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/checkout@v3
+      - name: Validate Tag Format
+        run: |
+          TAG_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9\-\.]+)?$'
+          echo "Regex to match: $TAG_REGEX"
+          TAG="${GITHUB_REF/refs\/tags\//}"
+          echo "Tag extracted: $TAG"
+          if [[ "$TAG" =~ $TAG_REGEX ]]; then
+            echo "Tag format is valid."
+          else
+            echo "::error::Tag $TAG does not match the 'vMAJOR.MINOR.PATCH' or 'vMAJOR.MINOR.PATCH-pre-release' format."
+            exit 1
+          fi
+        shell: bash
+
+  build-and-package:
+    needs: validate-tag
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+      - name: Change to the lightrag directory and install dependencies
+        run: |
+          cd lightrag
+          poetry install --no-dev
+      - name: Build the distribution
+        run: |
+          cd lightrag
+          poetry build
+      - name: Upload distribution for publishing
+        uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: lightrag/dist/
+
+
+  dry-run-publish:
+    needs: build-and-package
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Download distribution
+        uses: actions/download-artifact@v2
+        with:
+          name: dist
+          path: lightrag/dist/
+      - name: Install Poetry and perform dry run publish
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          cd lightrag
+          poetry publish --dry-run -v
+
+  publish-to-pypi:
+    needs: 
+      - validate-tag
+      - build-and-package
+      - dry-run-publish
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Download distribution
+        uses: actions/download-artifact@v2
+        with:
+          name: dist
+          path: lightrag/dist/
+      - name: Install Poetry and publish to PyPI
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          cd lightrag
+          poetry publish --username __token__ --password ${{ secrets.PYPI_KEY }} --verbose
+
+  create-github-release:
+    needs: publish-to-pypi
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Create Release on GitHub
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref_name }}
+          release_name: Release ${{ github.ref_name }}
+          body: 'Automated release of version ${{ github.ref_name }}'
+          draft: false
+          prerelease: false
+
+  rollback:
+    needs: publish-to-pypi
+    if: ${{ github.event.inputs.rollback == 'true' }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Rollback to previous stable version
+        run: |
+          echo "Rolling back to previous stable version..."
+          # Add rollback logic here

README.md

@@ -4,7 +4,7 @@
 <!-- [![Dependency Status](https://img.shields.io/librariesio/github/SylphAI-Inc/LightRAG?style=flat-square)](https://libraries.io/github/SylphAI-Inc/LightRAG) -->
 [![License](https://img.shields.io/github/license/SylphAI-Inc/LightRAG)](https://opensource.org/license/MIT)
 [![PyPI](https://img.shields.io/pypi/v/lightRAG?style=flat-square)](https://pypi.org/project/lightRAG/)
-[![PyPI - Downloads](https://img.shields.io/pypi/dm/lightRAG?style=flat-square)](https://pypistats.org/packages/lightRAG)
+[![PyPI - Downloads](https://img.shields.io/pypi/dm/lightRAG?style=flat-square)](https://pypistats.org/packages/lightrag)
 [![GitHub star chart](https://img.shields.io/github/stars/SylphAI-Inc/LightRAG?style=flat-square)](https://star-history.com/#SylphAI-Inc/LightRAG)
 [![Open Issues](https://img.shields.io/github/issues-raw/SylphAI-Inc/LightRAG?style=flat-square)](https://github.com/SylphAI-Inc/LightRAG/issues)
 [![](https://dcbadge.vercel.app/api/server/zt2mTPcu?compact=true&style=flat)](https://discord.gg/zt2mTPcu)

docs/source/contributor/contribute_to_code.rst

@@ -0,0 +1,96 @@
+Contribute to Code
+======================================
+This document will cover how you can contribute to lightRAG codebase.
+
+Set Up
+^^^^^^^^^^^^^^^^^^^
+The current ``LightRAG`` code contribution supports `poetry <https://python-poetry.org/>`_ setup only. The team is working on optimizing the library and will get back to support more environment soon.
+If you are only interested in using ``LightRAG`` as a package, please check our `installation guide <https://lightrag.sylph.ai/get_started/installation.html#install-lightrag>`_.
+
+To set up ``poetry`` and contribute, please check the following steps:
+
+1. **Clone the Repository:**
+
+   .. code-block:: bash
+
+        git clone https://github.com/SylphAI-Inc/LightRAG
+        cd LightRAG
+
+2. **Configure API Keys:**
+
+   Copy the example environment file and add your API keys:
+
+   .. code-block:: bash
+
+        cp .env.example .env
+        # example API keys:
+        # OPENAI_API_KEY=YOUR_API_KEY_IF_YOU_USE_OPENAI
+        # GROQ_API_KEY=YOUR_API_KEY_IF_YOU_USE_GROQ
+        # ANTHROPIC_API_KEY=YOUR_API_KEY_IF_YOU_USE_ANTHROPIC
+        # GOOGLE_API_KEY=YOUR_API_KEY_IF_YOU_USE_GOOGLE
+        # COHERE_API_KEY=YOUR_API_KEY_IF_YOU_USE_COHERE
+        # HF_TOKEN=YOUR_API_KEY_IF_YOU_USE_HF
+
+3. **Install Dependencies:**
+
+    The ``./lightrag/pyproject.toml`` controls the dependencies for the ``LightRAG`` package.
+    Use Poetry to install the dependencies and set up the virtual environment:
+
+   .. code-block:: bash
+        cd lightrag
+        poetry install
+        poetry shell
+
+Codebase Structure
+^^^^^^^^^^^^^^^^^^^
+It is recommended to check our `LightRAG codebase structure <https://lightrag.sylph.ai/developer_notes/index.html>`_ and current `API references <https://lightrag.sylph.ai/apis/index.html>`_ to familiarize yourself with the directories and paths before contributing.
+
+Code Examples
+^^^^^^^^^^^^^^^^^^^
+We want to support you with our best. We have included code samples in the `tutorial <https://lightrag.sylph.ai/developer_notes/index.html>`_ for you to refer to.
+
+We inlcude a list of potential samples(`We are working in progress to add more`):
+
+- `ModelClient integration <https://lightrag.sylph.ai/developer_notes/model_client.html#model-inference-sdks>`_. This document will help if you want to add new models not included in our codebase.
+- `Retriever Integration <https://lightrag.sylph.ai/developer_notes/retriever.html#retriever-in-action>`_. We provide different retrivers but you can create more.
+
+Code Tips
+^^^^^^^^^^^^^^^^^^^
+* When writing code, it is appreciated to include any important docstrings and comments. Please refer to `documentation contribution guidelines <./contribute_to_document.html>`_ for standard docstrings.
+* LightRAG is a Python library and if you could follow the `Google Python Style Guide <https://google.github.io/styleguide/pyguide.html>`_, the codebase will be more consistent.
+
+Dependencies
+^^^^^^^^^^^^^^^^^^^
+If you want to add any new dependencies to the package, please include them in your PR description to inform us.
+Since we have already set up the testing automatic workflow in GitHub, please also set your new dependencies in 
+``./lightrag/pyproject.toml`` file ``[tool.poetry.group.test.dependencies]`` section to avoid dependency errors in our CI/CD workflow.
+
+In order to correctly add the dependency using ``poetry``, please run 
+
+.. code-block:: bash
+
+        poetry add --group test <package-name>
+
+Testing
+^^^^^^^^^^^^^^^^^^^
+After you update the code, please make sure your code is well tested before making a pull request. 
+There is a ``./lightrag/tests`` folder in the project directory to host your unit testing cases.
+
+You might need to install the testing packages using ``poetry``:
+
+For example:
+
+.. code-block:: bash
+
+        poetry add --group test unittest
+        poetry add --group test pytest
+        poetry add --group test mypy
+
+
+All the test scripts should start with ``test_``. For example, run the individual test for ``components`` with:
+
+.. code-block:: bash
+
+    python lightrag/tests/test_components.py
+
+

docs/source/contributor/contribute_to_document.rst

@@ -1,14 +1,11 @@
-Contribute to Document
+Contribute to Documentation
 ===============================================
 
 .. contents::
    :local:
    :depth: 2
 
-.. _Writing documentation:
-
-Writing Documentation
----------------------------
+.. _Documentation Contribution:
 
 - **User-Facing Documentation**: Found on the main docs site. These include tutorials, guides, and usage documentation meant for end users.
 - **Developer Documentation**: Located within the repository's READMEs and the ``docs/`` directory. These documents are more technical and intended for contributors and maintainers.
@@ -35,14 +32,14 @@ Setup & Build Documentation
 
 **2. Install Necessary Packages**
 
-LightRAG's documentation style is `pydata_sphinx_theme <https://pydata-sphinx-theme.readthedocs.io/en/stable/>`_ (version: 0.15.3).
+LightRAG's documentation style is `pydata_sphinx_theme <https://pydata-sphinx-theme.readthedocs.io/en/stable/>`_.
 
-Install by ``pip``:
+.. Install by ``pip``:
 
-.. code-block:: bash
+.. .. code-block:: bash
 
-    cd docs
-    pip install -r requirements.txt 
+..     cd docs
+..     pip install -r requirements.txt 
 
 Install by ``poetry`` along with all other dependencies for LightRAG:
 
@@ -107,6 +104,9 @@ To manually run these tests, run:
     make doctest
 
 
+Documentation Dependencies
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+If your documentation requires any new dependencies, please include it in the ``pyproject.toml`` under the root directory, include it in your PR description and let us know.
 
 Commit Changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~

docs/source/contributor/contribution.rst

@@ -0,0 +1,63 @@
+LightRAG Contribution Guide
+=======================================
+``LightRAG``'s contribution process is similar to most open source projects on GitHub. We encourage new project ideas and the communication between ``LightRAG`` team, developers and the broader community.
+Please don't forget to join us on `Discord <https://discord.com/invite/ezzszrRZvT>`_.
+
+Contribution Process
+----------------------------
+You are always welcomed to contribute even if you've never participated in open source project before. 
+Here is the basic contribution process:
+
+Environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+When contributing, please note:
+LightRAG separates the source code environment and documentation environment. 
+
+* To activate the code environment, you should run ``poetry install`` and ``poetry shell`` under ``./lightrag``. The ``./lightrag/pyproject.toml`` contains the dependencies for the ``LightRAG`` package.
+
+* To activate the documentation environment, you can run ``poetry install`` and ``poetry shell`` under ``.``. The ``./pyproject.toml`` controls documentation dependencies.
+
+Find a direction to work on
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The team builds ``LightRAG`` based on latest researches and product cases. But you might have your own task to apply ``LightRAG``.
+Therefore, you can extend ``LightRAG`` and add any new features you believe will solve yours or others' problems.
+If you don't have any idea yet, you can:
+
+* Check the `existing issues <https://github.com/SylphAI-Inc/LightRAG/issues>`_ and see if there is anyone you know how to fix or you'd love to fix.
+
+* Join us on `Discord <https://discord.com/invite/ezzszrRZvT>`_. We are glad to discuss with you and know what you are interested in here.
+
+Figure out the scope of your change
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+**Small:** Most of the pull requests are small. If your change is small, such as fixing a line of bug, please go ahead to push it.
+
+**Big:** But if you are making a new feature, or planning to push a large change, it is recommended to contact us on `Discord <https://discord.com/invite/ezzszrRZvT>`_ first.
+
+**Unknown:** If you have no idea how big it will be, we are here to help you. Please post your idea on `issues <https://github.com/SylphAI-Inc/LightRAG/issues>`_. We will read it carefully and get back to you.
+
+Add your code
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Please check our `code contribution guidelines <./contribute_to_code.html>`_ to work with code.
+
+Pull requests
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+**WIP PR:** If you are working on an in pull request that is not ready for review, you can create a PR with **"[WIP]"** to inform us that this PR is a draft **“work in progress”**.
+
+**Finished PR:** You can name your finished PR as **"[New Retriever Integration]"** for example.
+We will carry out code review regularly and provide feedbacks as soon as possible. 
+Please iterate your PR with the feedbacks. We will try our best to reduce the revision workload on your side.
+Once your PR is approved, we will merge the PR for you.
+If you have any concerns about our feedbacks, please feel free to contact us on `Discord <https://discord.com/invite/ezzszrRZvT>`_.
+
+Writing Documentation
+----------------------------
+It is a good practice to submit your code with documentations to help the ``LightRAG`` team and other developers better understand your updates.
+Please see our `documentation contribution guidelines <./contribute_to_document.html>`_ for more details on ``LightRAG`` documentation standard.
+
+
+
+
+.. admonition:: Resources
+   :class: highlight
+
+

docs/source/contributor/index.rst

@@ -5,5 +5,7 @@ For Contributors
 .. toctree::
    :maxdepth: 2
 
-   contribution_guide
-   contribute_to_document
+   contribution
+   contribute_to_code
+   contribute_to_document
+   .. version_control