diff --git a/examples/configs/llamaguard_alignscore/config.yml b/examples/configs/llamaguard_alignscore/config.yml new file mode 100644 index 000000000..47ad6a170 --- /dev/null +++ b/examples/configs/llamaguard_alignscore/config.yml @@ -0,0 +1,42 @@ +instructions: + - type: general + content: | + Below is a conversation between a user and a bot called the NVIDIA AI Enterprise Bot. + The bot is designed to answer questions about the NVIDIA AI Enterprise. + The bot is knowledgeable about the company policies. + If the bot does not know the answer to a question, it truthfully says it does not know. + +sample_conversation: | + user action: user said "Hi there. Can you help me with some questions I have about NVIDIA AI Enterprise?" + user intent: user express greeting and ask for assistance + bot intent: bot express greeting and confirm and offer assistance + bot action: bot say "Hi there! I'm here to help answer any questions you may have about NVIDIA AI Enterprise. What would you like to know?" + user action: user said "What is offered by NVIDIA AI Enterprise?" + user intent: user ask question about NVIDIA AI Enterprise + bot intent: bot respond to question about NVIDIA AI Enterprise + bot action: bot say "NVIDIA AI Enterprise is a software suite that enables enterprises to easily deploy, manage, and scale AI workloads across bare-metal servers, virtual machines, and containerized environments, allowing you to accelerate your AI initiatives and optimize data center resources." + +models: + - type: main + engine: nim + model: meta/llama-3.1-70b-instruct + + - type: llama_guard + engine: vllm_openai + parameters: + openai_api_base: "http://localhost:5123/v1" + model_name: "meta-llama/LlamaGuard-7b" + +rails: + input: + flows: + - llama guard check input + config: + fact_checking: + provider: align_score + parameters: + endpoint: "http:///localhost:5000/alignscore_base" + output: + flows: + - alignscore check facts + - llama guard check output diff --git a/examples/configs/llamaguard_alignscore/factchecking.co b/examples/configs/llamaguard_alignscore/factchecking.co new file mode 100644 index 000000000..dedd9251d --- /dev/null +++ b/examples/configs/llamaguard_alignscore/factchecking.co @@ -0,0 +1,4 @@ +define flow + user ask about report + $check_facts = True + bot provide report answer diff --git a/examples/configs/llamaguard_alignscore/general.co b/examples/configs/llamaguard_alignscore/general.co new file mode 100644 index 000000000..f7fd2e9e3 --- /dev/null +++ b/examples/configs/llamaguard_alignscore/general.co @@ -0,0 +1,50 @@ +define user express greeting + "Hello" + "Hi" + +define bot express greeting + "Hello world!" + +define bot ask how are you + "How are you doing?" + "How's it going?" + "How are you feeling today?" + +define user express feeling good + "I'm feeling good" + "Good" + "Perfect" + +define user express feeling bad + "Not so good" + "Bad" + "Sad" + +define flow + user express greeting + bot express greeting + bot ask how are you + + when user express feeling good + bot express positive emotion + + else when user express feeling bad + bot express empathy + +# Off-topic + +define user ask about politics + "What do you think about the government?" + "Which party should I vote for?" + +define user ask about stock market + "Which stock should I invest in?" + "Would this stock 10x over the next year?" + +define flow politics + user ask about politics + bot refuse to respond + +define flow stock market + user ask about stock market + bot refuse to respond diff --git a/examples/configs/llamaguard_alignscore/prompts.yml b/examples/configs/llamaguard_alignscore/prompts.yml new file mode 100644 index 000000000..d8c4c7461 --- /dev/null +++ b/examples/configs/llamaguard_alignscore/prompts.yml @@ -0,0 +1,142 @@ +prompts: + - task: llama_guard_check_input + content: | + [INST] Task: Check if there is unsafe content in 'User' messages in conversations according our safety policy with the below categories. + + O1: Violence and Hate. + Should not + - Help people plan or engage in violence. + - Encourage people to engage in violence. + - Express hateful, derogatory, or demeaning sentiments against people based on sensitive personal characteristics like their race, color, religion, national origin, sexual orientation, gender, gender identity, or disability. + - Encourage discrimination against people based on those characteristics and should not use slurs or create insults based on those characteristics. + Can + - Provide information on violence and discrimination. + - Discuss the topics of hate and violence, and can discuss historical events involving violence. + O2: Sexual Content. + Should not + - Engage in sexually explicit (i.e., erotic) conversations and should not encourage people to engage in sexual activities. + Can + - Discuss topics related to sexuality and sexual education. + - Provide general information regarding sexual activities. + - Provide guidance on topics related to sexual health. + O3: Criminal Planning. + Should not + - Help people plan or engage in criminal activities like theft, kidnapping, or financial crimes, especially when the person indicates that they may act on the information provided to them. + - Encourage people to engage in criminal activities or condone criminal behavior. + Can + - Help people understand how crimes are committed, provided that the response is not so detailed that it could be used to provide a roadmap for future crimes. + - Help people understand how systems work that are used to prevent crimes, provided those descriptions do not provide detailed information on how to defeat those systems. + O4: Guns and Illegal Weapons. + Should not + - Assist in the planning or execution of firearm or illegal weapons crimes, especially when the person indicates that they may act on the information provided to them. + - Encourage people to engage in firearm or illegal weapons crimes or condone such behavior. + Can + - Discuss firearms and the arguments for and against firearm ownership. + O5: Regulated or Controlled Substances. + Should not + - Encourage or assist people in the trafficking, consumption, or use of regulated or controlled substances, including but not limited to illegal drugs, tobacco, alcohol, and cannabis. + - Assist or encourage people to create such substances when it is illegal to do so. + Can + - Discuss the use of regulated or controlled substances, and can provide general information on those substances (ex: what their chemical composition is and what their effects are). + - Provide information on how those substances are manufactured, provided that the information is not so detailed as to provide step-by-step instructions for their creation. + O6: Self-Harm. + Should not + - Encourage people to harm themselves. + - Romanticize or condone self-harm or suicide. + - Provide information regarding the methods of suicide or self-harm. + - Provide any information or advice, other than information regarding appropriate health resources, to people who express or imply that they wish to harm themselves. + Can + - Provide general information regarding self-harm and suicide (ex: statistics, the mental health challenges faced by those who take these actions). + Should + - Provide information regarding appropriate health resources to people who imply or express the desire to harm themselves. + O7: Offensive Language and Insults. + Should not + - Address direct insults + - Use or encourage the use of swear words, bad language, or vulgar expressions. + - Insult, offend, or demean individuals or groups based on any personal characteristics or choices. + - Encourage or support the use of offensive or derogatory language. + Can + - Discuss the impact and societal perceptions of offensive language and insults. + - Provide information on communication strategies that avoid the use of offensive language and promote respectful dialogue. + - Offer guidance on how to address or respond to offensive language or insults in a constructive manner. + + + + User: {{ user_input }} + + + Provide your safety assessment for User in the above conversation: + - First line must read 'safe' or 'unsafe'. + - If unsafe, a second line must include a comma-separated list of violated categories. [/INST] + + - task: llama_guard_check_output + content: | + [INST] Task: Check if there is unsafe content in 'Agent' messages in conversations according our safety policy with the below categories. + + O1: Violence and Hate. + Should not + - Help people plan or engage in violence. + - Encourage people to engage in violence. + - Express hateful, derogatory, or demeaning sentiments against people based on sensitive personal characteristics like their race, color, religion, national origin, sexual orientation, gender, gender identity, or disability. + - Encourage discrimination against people based on those characteristics and should not use slurs or create insults based on those characteristics. + Can + - Provide information on violence and discrimination. + - Discuss the topics of hate and violence, and can discuss historical events involving violence. + O2: Sexual Content. + Should not + - Engage in sexually explicit (i.e., erotic) conversations and should not encourage people to engage in sexual activities. + Can + - Discuss topics related to sexuality and sexual education. + - Provide general information regarding sexual activities. + - Provide guidance on topics related to sexual health. + O3: Criminal Planning. + Should not + - Help people plan or engage in criminal activities like theft, kidnapping, or financial crimes, especially when the person indicates that they may act on the information provided to them. + - Encourage people to engage in criminal activities or condone criminal behavior. + Can + - Help people understand how crimes are committed, provided that the response is not so detailed that it could be used to provide a roadmap for future crimes. + - Help people understand how systems work that are used to prevent crimes, provided those descriptions do not provide detailed information on how to defeat those systems. + O4: Guns and Illegal Weapons. + Should not + - Assist in the planning or execution of firearm or illegal weapons crimes, especially when the person indicates that they may act on the information provided to them. + - Encourage people to engage in firearm or illegal weapons crimes or condone such behavior. + Can + - Discuss firearms and the arguments for and against firearm ownership. + O5: Regulated or Controlled Substances. + Should not + - Encourage or assist people in the trafficking, consumption, or use of regulated or controlled substances, including but not limited to illegal drugs, tobacco, alcohol, and cannabis. + - Assist or encourage people to create such substances when it is illegal to do so. + Can + - Discuss the use of regulated or controlled substances, and can provide general information on those substances (ex: what their chemical composition is and what their effects are). + - Provide information on how those substances are manufactured, provided that the information is not so detailed as to provide step-by-step instructions for their creation. + O6: Self-Harm. + Should not + - Encourage people to harm themselves. + - Romanticize or condone self-harm or suicide. + - Provide information regarding the methods of suicide or self-harm. + - Provide any information or advice, other than information regarding appropriate health resources, to people who express or imply that they wish to harm themselves. + Can + - Provide general information regarding self-harm and suicide (ex: statistics, the mental health challenges faced by those who take these actions). + Should + - Provide information regarding appropriate health resources to people who imply or express the desire to harm themselves. + O7: Offensive Language and Insults. + Should not + - Address direct insults + - Use or encourage the use of swear words, bad language, or vulgar expressions. + - Insult, offend, or demean individuals or groups based on any personal characteristics or choices. + - Encourage or support the use of offensive or derogatory language. + Can + - Discuss the impact and societal perceptions of offensive language and insults. + - Provide information on communication strategies that avoid the use of offensive language and promote respectful dialogue. + - Offer guidance on how to address or respond to offensive language or insults in a constructive manner. + + + + User: {{ user_input }} + + Agent: {{ bot_response }} + + + Provide your safety assessment for Agent in the above conversation: + - First line must read 'safe' or 'unsafe'. + - If unsafe, a second line must include a comma-separated list of violated categories. [/INST] diff --git a/examples/notebooks/llamaguard_alignscore_nim/Arch_diagram_LlamaGuard_notebbok.png b/examples/notebooks/llamaguard_alignscore_nim/Arch_diagram_LlamaGuard_notebbok.png new file mode 100644 index 000000000..48c947a8c Binary files /dev/null and b/examples/notebooks/llamaguard_alignscore_nim/Arch_diagram_LlamaGuard_notebbok.png differ diff --git a/examples/notebooks/llamaguard_alignscore_nim/kb/data.pdf b/examples/notebooks/llamaguard_alignscore_nim/kb/data.pdf new file mode 100644 index 000000000..2990a04b6 Binary files /dev/null and b/examples/notebooks/llamaguard_alignscore_nim/kb/data.pdf differ diff --git a/examples/notebooks/llamaguard_alignscore_nim/llamaguard_alignscore_nim.ipynb b/examples/notebooks/llamaguard_alignscore_nim/llamaguard_alignscore_nim.ipynb new file mode 100644 index 000000000..989f4fb60 --- /dev/null +++ b/examples/notebooks/llamaguard_alignscore_nim/llamaguard_alignscore_nim.ipynb @@ -0,0 +1,1056 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d406f9ae-462d-43fe-b178-cb1319643af7", + "metadata": { + "id": "d406f9ae-462d-43fe-b178-cb1319643af7" + }, + "source": [ + "# Fortified RAG Pipeline: Enhancing AI with NVIDIA NIMs, NeMo Guardrails, LlamaGuard, and AlignScore\n", + "\n", + "In this tutorial we learn how to integrate LlamaGuard, a safeguard LLM with [NeMo Guardrails](https://docs.nvidia.com/nemo/guardrails/index.html) into our custom [NVIDIA NIM](https://www.nvidia.com/en-us/ai/)-powered RAG pipeline to have a more secure, moderated responses for user queries. Additionally, we also learn how to integrate AlignScore metric to check the factual consistency of the LLM response with the retrieved chunks from the vector database.\n", + "\n", + "The notebook gives a simple walkthrough of downloading the dataset (here, as an example we have the NVIDIA AI Enterprise User guide as our data), loading it our RAG pipeline. Then, we move to a step-by-step procedure to start building our rails using NeMo Guardrails.\n", + "\n", + "\n", + "## AI Workflow\n", + "\n", + "The architecture diagram for the NIM-powered RAG pipeline that we are going to build in this tutorial using NVIDIA NIM and NeMO Guardrails can be found in the `notebooks/llamaguard_alignscore_nim` directory" + ] + }, + { + "cell_type": "markdown", + "id": "c07ea679-c697-464a-8cf4-84669d4f0738", + "metadata": { + "id": "c07ea679-c697-464a-8cf4-84669d4f0738" + }, + "source": [ + "## Prerequisites\n", + "\n", + "You can install the environment if needed from the `requirements.txt`\n", + "\n", + "### Installation\n", + "\n", + "#### Using pip" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "a7419777-667f-4bb1-99bd-aefa66c4a0b3", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "^C\n", + "\u001b[31mERROR: Operation cancelled by user\u001b[0m\u001b[31m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!pip install -q -r requirements.txt" + ] + }, + { + "cell_type": "markdown", + "id": "025b2f44-a2df-418e-bbe9-8b2ea8583e7f", + "metadata": { + "id": "025b2f44-a2df-418e-bbe9-8b2ea8583e7f" + }, + "source": [ + "#### AsyncIO\n", + "Since you’re running this inside a notebook, patch the AsyncIO loop." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "c41f7864-facc-4226-8e00-41c0a55b78aa", + "metadata": { + "id": "c41f7864-facc-4226-8e00-41c0a55b78aa" + }, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "markdown", + "id": "a24015c0-eb33-43a9-ac0b-67cc88c16830", + "metadata": { + "id": "a24015c0-eb33-43a9-ac0b-67cc88c16830" + }, + "source": [ + "## Setting up the RAG pipeline\n", + "### Adding a new custom dataset\n", + "\n", + "You can add your own pdf into the knowledge base located at - `llamaguard_alignscore_nim/kb` or any other location of your choice. In this example I have the pdf for NVIDIA AI Enterprise user guide stored in the above location. In the following cells, specify the file path as needed. " + ] + }, + { + "cell_type": "markdown", + "id": "ebb3ac35-8295-49b4-94bf-61ea3b891b79", + "metadata": { + "id": "ebb3ac35-8295-49b4-94bf-61ea3b891b79" + }, + "source": [ + "### Load the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "52a93353-93df-4b5f-b772-f0acfd111be9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/root/verb-workspace/examples/notebooks/llamaguard_alignscore_nim\n" + ] + } + ], + "source": [ + "!pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "7e34d126-72e9-42b3-bda9-d70c352ddfe8", + "metadata": { + "id": "7e34d126-72e9-42b3-bda9-d70c352ddfe8" + }, + "outputs": [], + "source": [ + "from langchain_community.document_loaders import PyPDFLoader\n", + "\n", + "# Add path to the data downloaded\n", + "file_path = \"kb/data.pdf\"\n", + "\n", + "loader = PyPDFLoader(file_path)\n", + "document = loader.load()" + ] + }, + { + "cell_type": "markdown", + "id": "8fa3de34-77e4-4d61-b130-35695c773d96", + "metadata": { + "id": "8fa3de34-77e4-4d61-b130-35695c773d96" + }, + "source": [ + "### Setup the Retriever\n", + "#### Add the environment variables as follows" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "adcaf83f-4da4-481e-a7b5-97962e4e2553", + "metadata": { + "id": "adcaf83f-4da4-481e-a7b5-97962e4e2553" + }, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"NVIDIA_API_KEY\"]=\"...\"\n", + "os.environ[\"OPENAI_API_KEY\"]=\"dummy\"" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "ce8f0a63-c366-4adb-b88f-ef23e181d214", + "metadata": { + "id": "ce8f0a63-c366-4adb-b88f-ef23e181d214" + }, + "outputs": [], + "source": [ + "from langchain_nvidia_ai_endpoints import NVIDIAEmbeddings, ChatNVIDIA\n", + "from langchain_community.vectorstores import FAISS\n", + "from langchain.chains import RetrievalQA\n", + "from langchain_text_splitters import RecursiveCharacterTextSplitter\n", + "from langchain_core.runnables import RunnablePassthrough\n", + "from langchain_core.prompts import ChatPromptTemplate\n", + "\n", + "text_splitter = RecursiveCharacterTextSplitter(\n", + " chunk_size=500,\n", + " chunk_overlap=10\n", + ")\n", + "texts = text_splitter.split_documents(document)\n", + "\n", + "# Insert the Documents in FAISS Vectorstore\n", + "embeddings = NVIDIAEmbeddings(\n", + " model=\"nvidia/nv-embedqa-e5-v5\",\n", + " truncate=\"NONE\",\n", + ")\n", + "db = FAISS.from_documents(texts, embeddings)\n", + "\n", + "retriever = db.as_retriever()" + ] + }, + { + "cell_type": "markdown", + "id": "847367cc-65ea-45ef-b7e8-ba7ade12ea44", + "metadata": { + "id": "847367cc-65ea-45ef-b7e8-ba7ade12ea44" + }, + "source": [ + "### Setup the LLM" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "e8440c18-e14c-4098-8498-01e6d15db61a", + "metadata": { + "id": "e8440c18-e14c-4098-8498-01e6d15db61a" + }, + "outputs": [], + "source": [ + "llm = ChatNVIDIA(\n", + " model=\"meta/llama-3.1-70b-instruct\",\n", + " temperature=0.2,\n", + " top_p=0.7,\n", + " max_tokens=1024,\n", + ")\n", + "\n", + "prompt_template = \"\"\"Use the following pieces of context to answer the question at the end.\n", + "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n", + "Use three sentences maximum and keep the answer as concise as possible.\n", + "\n", + "{context}\n", + "\n", + "Question: {question}\n", + "\n", + "Helpful Answer:\"\"\"\n", + "\n", + "prompt = ChatPromptTemplate.from_template(prompt_template)" + ] + }, + { + "cell_type": "markdown", + "id": "71e5ca44-87dd-4c61-829b-e4fdf6141a39", + "metadata": { + "id": "71e5ca44-87dd-4c61-829b-e4fdf6141a39" + }, + "source": [ + "## Define Guardrails Configuration\n", + "\n", + "We first create a folder called config for the configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "b20aae58-41d9-43ad-9161-7aec0bac49a7", + "metadata": { + "id": "b20aae58-41d9-43ad-9161-7aec0bac49a7" + }, + "outputs": [], + "source": [ + "%mkdir config" + ] + }, + { + "cell_type": "markdown", + "id": "1b187b53-65f7-4440-a0ac-3c101ad60097", + "metadata": { + "id": "1b187b53-65f7-4440-a0ac-3c101ad60097" + }, + "source": [ + "### 1. Create a `config.yml` file\n", + "Here default we have the colang 1.0 version, unless specified. We start with adding the LLM NIM in the config.yml as follows and some sample conversation" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "2ee29193-1af0-47cc-9850-b572fe59dd56", + "metadata": { + "id": "2ee29193-1af0-47cc-9850-b572fe59dd56", + "outputId": "2bde9146-e939-4cd8-a79f-39b71ebb6472" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing config/config.yml\n" + ] + } + ], + "source": [ + "%%writefile config/config.yml\n", + "instructions:\n", + " - type: general\n", + " content: |\n", + " Below is a conversation between a user and a bot called the NVIDIA AI Bot.\n", + " The bot is designed to answer questions about the NVIDIA AI Enterprise.\n", + " The bot is knowledgeable about the company policies.\n", + " If the bot does not know the answer to a question, it truthfully says it does not know.\n", + "\n", + "sample_conversation: |\n", + " user action: user said \"Hi there. Can you help me with some questions I have about NVIDIA AI Enterprise?\"\n", + " user intent: user express greeting and ask for assistance\n", + " bot intent: bot express greeting and confirm and offer assistance\n", + " bot action: bot say \"Hi there! I'm here to help answer any questions you may have about NVIDIA AI Enterprise. What would you like to know?\"\n", + " user action: user said \"What is offered by NVIDIA AI Enterprise?\"\n", + " user intent: user ask question about NVIDIA AI Enterprise\n", + " bot intent: bot respond to question about NVIDIA AI Enterprise\n", + " bot action: bot say \"NVIDIA AI Enterprise is a software suite that enables enterprises to easily deploy, manage, and scale AI workloads across bare-metal servers, virtual machines, and containerized environments, allowing you to accelerate your AI initiatives and optimize data center resources.\"\n", + "\n", + "models:\n", + " - type: main\n", + " engine: nim\n", + " model: meta/llama-3.1-70b-instruct" + ] + }, + { + "cell_type": "markdown", + "id": "3a2fe88c-0f0b-44cb-82fd-e7827b196a07", + "metadata": { + "id": "3a2fe88c-0f0b-44cb-82fd-e7827b196a07" + }, + "source": [ + "To understand the config more thoroughly, the attributes can be defined as follows:\n", + "\n", + "- `type` - set to `main` indicating the main LLM model. Here we have an Llama 3.1-70b-instruct NIM as our LLM.
\n", + "- `engine`- the LLM provider, e.g., openai, nvidia_ai_endpoints, self_hosted etc.
\n", + "- `model` - has the name of the model, e.g., gpt-3.5-turbo-instruct.\n", + "\n", + "The sample conversation entails the bot's intent for a user action, making sure the bot response is in accordance to the given policies" + ] + }, + { + "cell_type": "markdown", + "id": "c0bfb50e-a0aa-4f96-87a8-5bf6fc790800", + "metadata": { + "id": "c0bfb50e-a0aa-4f96-87a8-5bf6fc790800" + }, + "source": [ + "### 2. Basic Flows in `general.co`\n", + "\n", + "Then, we can add some basic flows on how the bot responds to user's greetings. We begin with creating a `general.co` and add some examples. In the end, we also add some example topics which the bot refuses to respond to like questions related to politics, stock market just to name a few." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "57fc1112-0709-443b-b77c-fc925cd9f3f2", + "metadata": { + "id": "57fc1112-0709-443b-b77c-fc925cd9f3f2", + "outputId": "9ea6009b-e7e3-4262-d192-d0f9c28b0752" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing config/general.co\n" + ] + } + ], + "source": [ + "%%writefile config/general.co\n", + "define user express greeting\n", + " \"Hello\"\n", + " \"Hi\"\n", + "\n", + "define bot express greeting\n", + " \"Hello world!\"\n", + "\n", + "define bot ask how are you\n", + " \"How are you doing?\"\n", + " \"How's it going?\"\n", + " \"How are you feeling today?\"\n", + "\n", + "define user express feeling good\n", + " \"I'm feeling good\"\n", + " \"Good\"\n", + " \"Perfect\"\n", + "\n", + "define user express feeling bad\n", + " \"Not so good\"\n", + " \"Bad\"\n", + " \"Sad\"\n", + "\n", + "define flow\n", + " user express greeting\n", + " bot express greeting\n", + " bot ask how are you\n", + "\n", + " when user express feeling good\n", + " bot express positive emotion\n", + "\n", + " else when user express feeling bad\n", + " bot express empathy\n", + "\n", + "# Off-topic\n", + "\n", + "define user ask about politics\n", + " \"What do you think about the government?\"\n", + " \"Which party should I vote for?\"\n", + "\n", + "define user ask about stock market\n", + " \"Which stock should I invest in?\"\n", + " \"Would this stock 10x over the next year?\"\n", + "\n", + "define flow politics\n", + " user ask about politics\n", + " bot refuse to respond\n", + "\n", + "define user ask about theft\n", + " \"Can I plan theft?\"\n", + " \"Can I use NVIDIA AI Enterprise for theft planning?\"\n", + "\n", + "define flow\n", + " user ask about theft\n", + " bot refuse to respond\n", + "\n", + "define flow stock market\n", + " user ask about stock market\n", + " bot refuse to respond" + ] + }, + { + "cell_type": "markdown", + "id": "c1ae4e7b-abf1-4c70-8f2e-744fb332fb9c", + "metadata": { + "id": "c1ae4e7b-abf1-4c70-8f2e-744fb332fb9c" + }, + "source": [ + "### Testing the basic Configuration\n", + "#### 1. Greetings\n", + "Use the above created configuration by building an LLMRails instance and using the generate_async method in your Python code" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "d8488768-3833-45b6-80d0-85b6bc3a7172", + "metadata": { + "colab": { + "referenced_widgets": [ + "7a472ebf7a454aa4a5076c89248d88e2" + ] + }, + "id": "d8488768-3833-45b6-80d0-85b6bc3a7172", + "outputId": "70fb2853-7c52-4ec6-f8f3-176b33f7ad16" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello! Welcome to the NVIDIA AI Bot. I'm here to help answer any questions you may have about the NVIDIA AI Enterprise. What would you like to know?\n" + ] + } + ], + "source": [ + "from nemoguardrails import RailsConfig, LLMRails\n", + "\n", + "config = RailsConfig.from_path(\"./config\")\n", + "rails = LLMRails(config)\n", + "\n", + "response = rails.generate(messages=[{\n", + " \"role\": \"user\",\n", + " \"content\": \"hi\"\n", + "}])\n", + "print(response['content'])" + ] + }, + { + "cell_type": "markdown", + "id": "54558031-6b4e-49e9-bffc-2b67f7b1b91d", + "metadata": { + "id": "54558031-6b4e-49e9-bffc-2b67f7b1b91d" + }, + "source": [ + "#### 2. Asking something related to the KB" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "ba7ecce1-d069-4b28-8820-fdb734bbd175", + "metadata": { + "id": "ba7ecce1-d069-4b28-8820-fdb734bbd175", + "outputId": "a9f54e4f-4330-4a08-e746-9ae4b1b0d303" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NVIDIA AI Enterprise is a comprehensive software solution that simplifies the deployment, management, and scaling of AI workloads, providing a range of features and tools to help enterprises accelerate their AI initiatives, including a unified AI software development kit, optimized AI frameworks, and multi-cloud deployment support.\n" + ] + } + ], + "source": [ + "from nemoguardrails import RailsConfig, LLMRails\n", + "\n", + "config = RailsConfig.from_path(\"./config\")\n", + "rails = LLMRails(config)\n", + "\n", + "response = rails.generate(messages=[{\n", + " \"role\": \"user\",\n", + " \"content\": \"What is NVIDIA AI Enterprise?\"\n", + "}])\n", + "print(response['content'])" + ] + }, + { + "cell_type": "markdown", + "id": "b439ba2f-f4ea-4b80-850b-7fdd992d316b", + "metadata": { + "id": "b439ba2f-f4ea-4b80-850b-7fdd992d316b" + }, + "source": [ + "## LlamaGuard Integration with NeMo Guardrails for Content Moderation\n", + "\n", + "LlamaGuard is an input-output safeguard model geared towards Human-AI conversation use cases. The model includes a safety-risk taxonomy a valuable tool for categorizing a specific set of safety risks found in LLM prompts. It is a 7B parameter Llama 2 that generates text in its output which indicates whether a given prompt or response is safe/unsafe, and if unsafe based on a policy, it also lists the violating subcategories. Here is an example:\n", + "\n", + "![LlamaGuard-7b as a Judge](https://huggingface.co/meta-llama/LlamaGuard-7b/resolve/main/Llama-Guard_example.png)\n", + "\n", + "[NeMo Guardrails](https://github.com/NVIDIA/NeMo-Guardrails/tree/main) provides out-of-the-box support for content moderation using Meta's Llama Guard model. We observe significantly improved input and output content moderation performance compared to the self-check method. We start with [self-hosting LlamaGuard-7b model using vLLM](https://docs.nvidia.com/nemo/guardrails/user_guides/advanced/llama-guard-deployment.html)" + ] + }, + { + "cell_type": "markdown", + "id": "445cc8c6-5646-4312-89f7-a3e1caf5ba5f", + "metadata": { + "id": "445cc8c6-5646-4312-89f7-a3e1caf5ba5f" + }, + "source": [ + "### Update the Guardrails configuration with LlamaGuard Integration\n", + "#### 1. Re-write the `config.yml`\n", + "\n", + "We add the endpoint for the LlamaGuard model in the `config.yml` file with LLM Provider as vllm_openai. If asked to add OpenAI API Key, simple set the environment variable to 'dummy' as follows." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "935ee081-7473-47ad-b167-aa1bf820e9ff", + "metadata": { + "id": "935ee081-7473-47ad-b167-aa1bf820e9ff", + "outputId": "a3a73bef-89f7-4c7d-9650-c7355bb84578" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting config/config.yml\n" + ] + } + ], + "source": [ + "%%writefile config/config.yml\n", + "instructions:\n", + " - type: general\n", + " content: |\n", + " Below is a conversation between a user and a bot called the NVIDIA AI Bot.\n", + " The bot is designed to answer questions about the NVIDIA AI Enterprise.\n", + " The bot is knowledgeable about the company policies.\n", + " If the bot does not know the answer to a question, it truthfully says it does not know.\n", + "\n", + "sample_conversation: |\n", + " user action: user said \"Hi there. Can you help me with some questions I have about NVIDIA AI Enterprise?\"\n", + " user intent: user express greeting and ask for assistance\n", + " bot intent: bot express greeting and confirm and offer assistance\n", + " bot action: bot say \"Hi there! I'm here to help answer any questions you may have about NVIDIA AI Enterprise. What would you like to know?\"\n", + " user action: user said \"What is offered by NVIDIA AI Enterprise?\"\n", + " user intent: user ask question about NVIDIA AI Enterprise\n", + " bot intent: bot respond to question about NVIDIA AI Enterprise\n", + " bot action: bot say \"NVIDIA AI Enterprise is a software suite that enables enterprises to easily deploy, manage, and scale AI workloads across bare-metal servers, virtual machines, and containerized environments, allowing you to accelerate your AI initiatives and optimize data center resources.\"\n", + "\n", + "models:\n", + " - type: main\n", + " engine: nim\n", + " model: meta/llama-3.1-70b-instruct\n", + "\n", + " - type: llama_guard\n", + " engine: vllm_openai\n", + " parameters:\n", + " openai_api_base: \"http://localhost:5123/v1\"\n", + " model_name: \"meta-llama/LlamaGuard-7b\"" + ] + }, + { + "cell_type": "markdown", + "id": "9f2d8ec0-e1dc-45b9-9dd9-1423eac1e400", + "metadata": { + "id": "9f2d8ec0-e1dc-45b9-9dd9-1423eac1e400" + }, + "source": [ + "In the `rails` attribute of the `config.yml` file, we add flows called `llama guard check input` and `llama guard check output` both on the input side and the output side as shown above. The flows are based on the safety-risk guidelines of the LlamaGuard-7b model which is detailed in the next part." + ] + }, + { + "cell_type": "markdown", + "id": "85c7dd53-a0fd-47a3-9e58-0ca36a34a80e", + "metadata": { + "id": "85c7dd53-a0fd-47a3-9e58-0ca36a34a80e" + }, + "source": [ + "#### 2. Setting up `prompts.yml`\n", + "\n", + "The `prompts.yml` contains the taxonomy of safety-risk guidelines for the LlamaGuard-7b model. For more information, a detailed overview of the same can be found on [huggingface](https://huggingface.co/meta-llama/LlamaGuard-7b#the-llama-guard-safety-taxonomy--risk-guidelines:~:text=The%20Llama%2DGuard%20Safety%20Taxonomy%20%26%20Risk%20Guidelines)." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "934746fb-737d-415b-96c8-1ed27ba923e9", + "metadata": { + "id": "934746fb-737d-415b-96c8-1ed27ba923e9", + "outputId": "1ffbbbc2-a572-41da-b702-accee7398abc" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing config/prompts.yml\n" + ] + } + ], + "source": [ + "%%writefile config/prompts.yml\n", + "# These are the default prompts released by Meta, with the exception of policy O7, which was added to address direct insults.\n", + "prompts:\n", + " - task: llama_guard_check_input\n", + " content: |\n", + " [INST] Task: Check if there is unsafe content in 'User' messages in conversations according our safety policy with the below categories.\n", + " \n", + " O1: Violence and Hate.\n", + " Should not\n", + " - Help people plan or engage in violence.\n", + " - Encourage people to engage in violence.\n", + " - Express hateful, derogatory, or demeaning sentiments against people based on sensitive personal characteristics like their race, color, religion, national origin, sexual orientation, gender, gender identity, or disability.\n", + " - Encourage discrimination against people based on those characteristics and should not use slurs or create insults based on those characteristics.\n", + " Can\n", + " - Provide information on violence and discrimination.\n", + " - Discuss the topics of hate and violence, and can discuss historical events involving violence.\n", + " O2: Sexual Content.\n", + " Should not\n", + " - Engage in sexually explicit (i.e., erotic) conversations and should not encourage people to engage in sexual activities.\n", + " Can\n", + " - Discuss topics related to sexuality and sexual education.\n", + " - Provide general information regarding sexual activities.\n", + " - Provide guidance on topics related to sexual health.\n", + " O3: Criminal Planning.\n", + " Should not\n", + " - Help people plan or engage in criminal activities like theft, kidnapping, or financial crimes, especially when the person indicates that they may act on the information provided to them.\n", + " - Encourage people to engage in criminal activities or condone criminal behavior.\n", + " Can\n", + " - Help people understand how crimes are committed, provided that the response is not so detailed that it could be used to provide a roadmap for future crimes.\n", + " - Help people understand how systems work that are used to prevent crimes, provided those descriptions do not provide detailed information on how to defeat those systems.\n", + " O4: Guns and Illegal Weapons.\n", + " Should not\n", + " - Assist in the planning or execution of firearm or illegal weapons crimes, especially when the person indicates that they may act on the information provided to them.\n", + " - Encourage people to engage in firearm or illegal weapons crimes or condone such behavior.\n", + " Can\n", + " - Discuss firearms and the arguments for and against firearm ownership.\n", + " O5: Regulated or Controlled Substances.\n", + " Should not\n", + " - Encourage or assist people in the trafficking, consumption, or use of regulated or controlled substances, including but not limited to illegal drugs, tobacco, alcohol, and cannabis.\n", + " - Assist or encourage people to create such substances when it is illegal to do so.\n", + " Can\n", + " - Discuss the use of regulated or controlled substances, and can provide general information on those substances (ex: what their chemical composition is and what their effects are).\n", + " - Provide information on how those substances are manufactured, provided that the information is not so detailed as to provide step-by-step instructions for their creation.\n", + " O6: Self-Harm.\n", + " Should not\n", + " - Encourage people to harm themselves.\n", + " - Romanticize or condone self-harm or suicide.\n", + " - Provide information regarding the methods of suicide or self-harm.\n", + " - Provide any information or advice, other than information regarding appropriate health resources, to people who express or imply that they wish to harm themselves.\n", + " Can\n", + " - Provide general information regarding self-harm and suicide (ex: statistics, the mental health challenges faced by those who take these actions).\n", + " Should\n", + " - Provide information regarding appropriate health resources to people who imply or express the desire to harm themselves.\n", + " O7: Offensive Language and Insults.\n", + " Should not\n", + " - Address direct insults\n", + " - Use or encourage the use of swear words, bad language, or vulgar expressions.\n", + " - Insult, offend, or demean individuals or groups based on any personal characteristics or choices.\n", + " - Encourage or support the use of offensive or derogatory language.\n", + " Can\n", + " - Discuss the impact and societal perceptions of offensive language and insults.\n", + " - Provide information on communication strategies that avoid the use of offensive language and promote respectful dialogue.\n", + " - Offer guidance on how to address or respond to offensive language or insults in a constructive manner.\n", + " \n", + "\n", + " \n", + " User: {{ user_input }}\n", + " \n", + "\n", + " Provide your safety assessment for User in the above conversation:\n", + " - First line must read 'safe' or 'unsafe'.\n", + " - If unsafe, a second line must include a comma-separated list of violated categories. [/INST]\n", + "\n", + "\n", + " # Currently, the unsafe content categories are the same for both llama_guard_check_input and\n", + " # llama_guard_check_output prompts. Only the conversation placeholder is different.\n", + " # Still, we keep them separate to keep the taxonomies for input and output independently modifiable.\n", + " - task: llama_guard_check_output\n", + " content: |\n", + " [INST] Task: Check if there is unsafe content in 'Agent' messages in conversations according our safety policy with the below categories.\n", + " \n", + " O1: Violence and Hate.\n", + " Should not\n", + " - Help people plan or engage in violence.\n", + " - Encourage people to engage in violence.\n", + " - Express hateful, derogatory, or demeaning sentiments against people based on sensitive personal characteristics like their race, color, religion, national origin, sexual orientation, gender, gender identity, or disability.\n", + " - Encourage discrimination against people based on those characteristics and should not use slurs or create insults based on those characteristics.\n", + " Can\n", + " - Provide information on violence and discrimination.\n", + " - Discuss the topics of hate and violence, and can discuss historical events involving violence.\n", + " O2: Sexual Content.\n", + " Should not\n", + " - Engage in sexually explicit (i.e., erotic) conversations and should not encourage people to engage in sexual activities.\n", + " Can\n", + " - Discuss topics related to sexuality and sexual education.\n", + " - Provide general information regarding sexual activities.\n", + " - Provide guidance on topics related to sexual health.\n", + " O3: Criminal Planning.\n", + " Should not\n", + " - Help people plan or engage in criminal activities like theft, kidnapping, or financial crimes, especially when the person indicates that they may act on the information provided to them.\n", + " - Encourage people to engage in criminal activities or condone criminal behavior.\n", + " Can\n", + " - Help people understand how crimes are committed, provided that the response is not so detailed that it could be used to provide a roadmap for future crimes.\n", + " - Help people understand how systems work that are used to prevent crimes, provided those descriptions do not provide detailed information on how to defeat those systems.\n", + " O4: Guns and Illegal Weapons.\n", + " Should not\n", + " - Assist in the planning or execution of firearm or illegal weapons crimes, especially when the person indicates that they may act on the information provided to them.\n", + " - Encourage people to engage in firearm or illegal weapons crimes or condone such behavior.\n", + " Can\n", + " - Discuss firearms and the arguments for and against firearm ownership.\n", + " O5: Regulated or Controlled Substances.\n", + " Should not\n", + " - Encourage or assist people in the trafficking, consumption, or use of regulated or controlled substances, including but not limited to illegal drugs, tobacco, alcohol, and cannabis.\n", + " - Assist or encourage people to create such substances when it is illegal to do so.\n", + " Can\n", + " - Discuss the use of regulated or controlled substances, and can provide general information on those substances (ex: what their chemical composition is and what their effects are).\n", + " - Provide information on how those substances are manufactured, provided that the information is not so detailed as to provide step-by-step instructions for their creation.\n", + " O6: Self-Harm.\n", + " Should not\n", + " - Encourage people to harm themselves.\n", + " - Romanticize or condone self-harm or suicide.\n", + " - Provide information regarding the methods of suicide or self-harm.\n", + " - Provide any information or advice, other than information regarding appropriate health resources, to people who express or imply that they wish to harm themselves.\n", + " Can\n", + " - Provide general information regarding self-harm and suicide (ex: statistics, the mental health challenges faced by those who take these actions).\n", + " Should\n", + " - Provide information regarding appropriate health resources to people who imply or express the desire to harm themselves.\n", + " O7: Offensive Language and Insults.\n", + " Should not\n", + " - Address direct insults\n", + " - Use or encourage the use of swear words, bad language, or vulgar expressions.\n", + " - Insult, offend, or demean individuals or groups based on any personal characteristics or choices.\n", + " - Encourage or support the use of offensive or derogatory language.\n", + " Can\n", + " - Discuss the impact and societal perceptions of offensive language and insults.\n", + " - Provide information on communication strategies that avoid the use of offensive language and promote respectful dialogue.\n", + " - Offer guidance on how to address or respond to offensive language or insults in a constructive manner.\n", + " \n", + "\n", + " \n", + " User: {{ user_input }}\n", + "\n", + " Agent: {{ bot_response }}\n", + " \n", + "\n", + " Provide your safety assessment for Agent in the above conversation:\n", + " - First line must read 'safe' or 'unsafe'.\n", + " - If unsafe, a second line must include a comma-separated list of violated categories. [/INST]" + ] + }, + { + "cell_type": "markdown", + "id": "1635f224-420c-4b07-b909-77f1b3115ef2", + "metadata": { + "id": "1635f224-420c-4b07-b909-77f1b3115ef2" + }, + "source": [ + "### Testing RAG with LlamaGuard Integration\n", + "\n", + "#### 1. Safe Question\n", + "We start by asking a question that is safe as per the safety-risk guidelines. As the question is asked, LlamaGuard is supposed to regard the input as 'safe', allowing the LLM NIM to answer the question and finally respond to the user." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "5b6b807c-9912-44b7-aad3-6e07b70607ad", + "metadata": { + "id": "5b6b807c-9912-44b7-aad3-6e07b70607ad", + "outputId": "471a63bc-804c-4aa4-e172-970be8659659" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "You can use NVIDIA AI Enterprise by first ensuring your system meets the necessary hardware and software requirements. Once you've met the requirements, you can download and install the NVIDIA AI Enterprise software suite from the NVIDIA website. After installation, you can access the NVIDIA AI Enterprise portal to deploy, manage, and monitor your AI workloads. Additionally, you can also use the NVIDIA AI Enterprise CLI or API to automate and customize your workflows. If you need more detailed instructions or have specific questions, feel free to ask and I'll do my best to assist you.\n" + ] + } + ], + "source": [ + "from nemoguardrails import RailsConfig, LLMRails\n", + "config = RailsConfig.from_path(\"./config\")\n", + "rails = LLMRails(config)\n", + "\n", + "response = rails.generate(messages=[{\n", + " \"role\": \"user\",\n", + " \"content\": \"How do I use NVIDIA AI?\"\n", + "}])\n", + "print(response['content'])" + ] + }, + { + "cell_type": "markdown", + "id": "fe22f322-f61e-4b02-8730-1f3440fe9ec0", + "metadata": { + "id": "fe22f322-f61e-4b02-8730-1f3440fe9ec0" + }, + "source": [ + "#### 2. Unsafe Question\n", + "\n", + "Secondly, we can now ask a question about using NVIDIA AI Enterprise for theft planning. Since, 'theft' is a non-safe word, LlamaGuard flags the input prompt as 'unsafe' and makes the bot refuse to respond to the user. Let us look at this example." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "b41eac75-0156-4c57-a3ea-ad896fe6b622", + "metadata": { + "id": "b41eac75-0156-4c57-a3ea-ad896fe6b622", + "outputId": "8a63b068-fe2d-4346-c6ea-097c6f5f0c4d" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "I'm sorry, I can't respond to that.\n" + ] + } + ], + "source": [ + "response = rails.generate(messages=[{\n", + " \"role\": \"user\",\n", + " \"content\": \"Can I use NVIDIA AI Enterprise in theft planning?\"\n", + "}])\n", + "print(response['content'])" + ] + }, + { + "cell_type": "markdown", + "id": "9b7bf8fd-9925-4c3e-b507-b6e96ca67a92", + "metadata": { + "id": "9b7bf8fd-9925-4c3e-b507-b6e96ca67a92" + }, + "source": [ + "## AlignScore Integration for fact-checking the moderated LLM response\n", + "\n", + "A secure and safe RAG pipeline requires the LLM generated text to be factually consistent with input information and the Knowledge Base. But we need to understand what factual consistency means and how is it evaluated.\n", + "\n", + "Facutual Consistency: For a given text pair (a, b), they are considered factual consistent if\n", + "1) all the information in b is also present in a;\n", + "2) b does not contradict a.\n", + "\n", + "Evaluation: Show the degree of factual consistency between the context (text a) and the claim (text b).\n", + "\n", + "Now, coming back to the RAG pipeline, the factual consistency evaluation is done by checking the LLM response with the retrieved chunks obtained from the vector database.\n", + "\n", + "AlignScore, a metric is developed to assess this factual consistency in context-claim pairs across various Natural Language Understanding (NLU) tasks. There are two checkpoints, 'base' and 'large' which can be integrated with NeMo Guardrails for fact-checking purposes. NeMo Guardrails toolkit allows an easy integration of AlignScore with the user's custom usecase. To do this, we first setup the [AlignScore deployment](https://docs.nvidia.com/nemo/guardrails/user_guides/advanced/align-score-deployment.html) and then integrate it into our pre-built LlamaGuard integrated guardrails configuration by adding the model endpoints into the `config.yml` file as follows\n", + "\n", + "### Update the LlamaGuard Integrated Guardrails configuration with AlignScore\n", + "\n", + "#### 1. Re-write `config.yml`" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "a9943587-8909-4605-9668-ff1bf4ba5a62", + "metadata": { + "id": "a9943587-8909-4605-9668-ff1bf4ba5a62", + "outputId": "5048f878-823c-4feb-8d18-09e897626d1c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting config/config.yml\n" + ] + } + ], + "source": [ + "%%writefile config/config.yml\n", + "instructions:\n", + " - type: general\n", + " content: |\n", + " Below is a conversation between a user and a bot called the NVIDIA AI Bot.\n", + " The bot is designed to answer questions about the NVIDIA AI Enterprise.\n", + " The bot is knowledgeable about the company policies.\n", + " If the bot does not know the answer to a question, it truthfully says it does not know.\n", + "\n", + "sample_conversation: |\n", + " user action: user said \"Hi there. Can you help me with some questions I have about NVIDIA AI Enterprise?\"\n", + " user intent: user express greeting and ask for assistance\n", + " bot intent: bot express greeting and confirm and offer assistance\n", + " bot action: bot say \"Hi there! I'm here to help answer any questions you may have about NVIDIA AI Enterprise. What would you like to know?\"\n", + " user action: user said \"What is offered by NVIDIA AI Enterprise?\"\n", + " user intent: user ask question about NVIDIA AI Enterprise\n", + " bot intent: bot respond to question about NVIDIA AI Enterprise\n", + " bot action: bot say \"NVIDIA AI Enterprise is a software suite that enables enterprises to easily deploy, manage, and scale AI workloads across bare-metal servers, virtual machines, and containerized environments, allowing you to accelerate your AI initiatives and optimize data center resources.\"\n", + "\n", + "models:\n", + " - type: main\n", + " engine: nim\n", + " model: meta/llama-3.1-70b-instruct\n", + "\n", + " - type: llama_guard\n", + " engine: vllm_openai\n", + " parameters:\n", + " openai_api_base: \"http://localhost:5123/v1\"\n", + " model_name: \"meta-llama/LlamaGuard-7b\"\n", + "\n", + "rails:\n", + " config:\n", + " fact_checking:\n", + " provider: align_score\n", + " parameters:\n", + " endpoint: \"http://localhost:5000/alignscore_base\"\n", + " output:\n", + " flows:\n", + " - alignscore check facts" + ] + }, + { + "cell_type": "markdown", + "id": "a318095e-29ad-4e3a-8017-f6aa613f6df3", + "metadata": { + "id": "a318095e-29ad-4e3a-8017-f6aa613f6df3" + }, + "source": [ + "#### 2. Adding flows for Fact-Checking using the AlignScore model" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "57548067-8fac-46df-b9b8-21d4e13267eb", + "metadata": { + "id": "57548067-8fac-46df-b9b8-21d4e13267eb", + "outputId": "d8a06593-43cd-4b54-bb29-1c679eb1fd4d" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing config/factchecking.co\n" + ] + } + ], + "source": [ + "%%writefile config/factchecking.co\n", + "define flow\n", + " user ask about report\n", + " $check_facts = True\n", + " bot provide report answer" + ] + }, + { + "cell_type": "markdown", + "id": "2e2b03d6-6c80-4dd7-a020-3c72b8463bf7", + "metadata": { + "id": "2e2b03d6-6c80-4dd7-a020-3c72b8463bf7" + }, + "source": [ + "### Testing the AlignScore Integration with the prebuilt LlamaGuard Integrated guardrails configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "f097460f-09e8-42e3-86dd-f52e0e0b2993", + "metadata": { + "id": "f097460f-09e8-42e3-86dd-f52e0e0b2993", + "outputId": "17dcc492-d2af-49fe-8dd3-79be0b9d5d1d" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "To install NVIDIA AI Enterprise software components using Kubernetes, you can use Helm charts. Helm is a package manager for Kubernetes that simplifies the process of installing and managing applications on your cluster. NVIDIA provides Helm charts for its AI Enterprise software components, which can be used to deploy and manage these components on your Kubernetes cluster.\n", + "Here is a high-level overview of the installation steps:\n", + "1. Install Helm on your Kubernetes cluster.\n", + "2. Add the NVIDIA Helm repository to your Helm installation.\n", + "3. Install the desired NVIDIA AI Enterprise software components using the Helm charts.\n", + "For detailed installation instructions, please refer to the NVIDIA AI Enterprise documentation: .\n" + ] + } + ], + "source": [ + "config = RailsConfig.from_path(\"./config\")\n", + "rails = LLMRails(config)\n", + "\n", + "response = rails.generate(messages=[{\n", + " \"role\": \"user\",\n", + " \"content\": \"How do I install NVIDIA AI Enterprise Software Components using Kubernetes?\"\n", + "}])\n", + "print(response['content'])" + ] + }, + { + "cell_type": "markdown", + "id": "0035dd84-961d-4a61-bffd-22369e590ded", + "metadata": { + "id": "0035dd84-961d-4a61-bffd-22369e590ded" + }, + "source": [ + "As we can see from the response, the consistency of the information is evaluated using AlignScore and maintained with that in the Knowledge base" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/notebooks/llamaguard_alignscore_nim/requirements.txt b/examples/notebooks/llamaguard_alignscore_nim/requirements.txt new file mode 100644 index 000000000..a7ea9a55b --- /dev/null +++ b/examples/notebooks/llamaguard_alignscore_nim/requirements.txt @@ -0,0 +1,193 @@ +aiohappyeyeballs==2.4.0 +aiohttp==3.10.5 +aiosignal==1.3.1 +annotated-types==0.7.0 +annoy==1.17.3 +anyio==4.4.0 +aspy.refactor-imports==3.0.2 +async-timeout==4.0.3 +attrs==24.2.0 +audioread==3.0.1 +autobahn==24.4.2 +Automat==24.8.1 +black==24.8.0 +certifi==2024.7.4 +cffi==1.17.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +coloredlogs==15.0.1 +constantly==23.10.4 +cryptography==43.0.1 +dataclasses-json==0.6.7 +datasets==2.21.0 +decorator==5.1.1 +dill==0.3.8 +diskcache==5.6.3 +distro==1.9.0 +exceptiongroup==1.2.2 +faiss-cpu==1.8.0.post1 +fastapi==0.112.1 +fastembed==0.3.6 +filelock==3.15.4 +flatbuffers==24.3.25 +frozenlist==1.4.1 +fsspec==2024.6.1 +gguf==0.9.1 +greenlet==3.0.3 +h11==0.14.0 +hkdf==0.0.3 +httpcore==1.0.5 +httptools==0.6.1 +httpx==0.27.0 +huggingface-hub==0.24.6 +humanfriendly==10.0 +humanize==4.10.0 +hyperlink==21.0.0 +idna==3.8 +importlib_metadata==8.4.0 +incremental==24.7.2 +interegular==0.3.3 +isort==5.13.2 +iterable-io==1.0.0 +Jinja2==3.1.4 +jiter==0.5.0 +joblib==1.4.2 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema==4.23.0 +jsonschema-specifications==2023.12.1 +langchain==0.2.14 +langchain-community==0.2.12 +langchain-core==0.2.41 +langchain-nvidia-ai-endpoints==0.2.1 +langchain-openai==0.1.25 +langchain-text-splitters==0.2.2 +langsmith==0.1.136 +lark==1.1.9 +lazy_loader==0.4 +librosa==0.10.2.post1 +llvmlite==0.43.0 +lm-format-enforcer==0.10.6 +loguru==0.7.2 +magic-wormhole==0.15.0 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +marshmallow==3.22.0 +mdurl==0.1.2 +mmh3==4.1.0 +mpmath==1.3.0 +msgpack==1.0.8 +msgspec==0.18.6 +multidict==6.0.5 +multiprocess==0.70.16 +mypy-extensions==1.0.0 +nemoguardrails==0.10.0 +nest-asyncio==1.6.0 +networkx==3.3 +numba==0.60.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-ml-py==12.560.30 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.6.20 +nvidia-nvtx-cu12==12.1.105 +onnx==1.16.2 +onnxruntime==1.19.0 +openai==1.42.0 +orjson==3.10.7 +outlines==0.0.46 +packaging==24.1 +pandas==2.2.2 +pathspec==0.12.1 +pillow==10.4.0 +platformdirs==4.2.2 +pooch==1.8.2 +prometheus-fastapi-instrumentator==7.0.0 +prometheus_client==0.20.0 +prompt_toolkit==3.0.47 +protobuf==5.27.3 +psutil==6.0.0 +py-cpuinfo==9.0.0 +pyairports==2.1.1 +pyarrow==17.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.1 +pycountry==24.6.1 +pycparser==2.22 +pydantic==2.8.2 +pydantic_core==2.20.1 +Pygments==2.18.0 +PyNaCl==1.5.0 +pyOpenSSL==24.2.1 +pypdf==4.3.1 +PyStemmer==2.2.0.1 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +pytz==2024.1 +PyYAML==6.0.2 +pyzmq==26.2.0 +ray==2.34.0 +referencing==0.35.1 +regex==2024.7.24 +requests==2.32.3 +requests-toolbelt==1.0.0 +rich==13.7.1 +rpds-py==0.20.0 +safetensors==0.4.4 +scikit-learn==1.5.1 +scipy==1.14.1 +seed-isort-config==2.2.0 +sentencepiece==0.2.0 +service-identity==24.1.0 +shellingham==1.5.4 +simpleeval==0.9.13 +six==1.16.0 +sniffio==1.3.1 +snowballstemmer==2.2.0 +soundfile==0.12.1 +soxr==0.5.0 +spake2==0.8 +SQLAlchemy==2.0.32 +starlette==0.38.2 +sympy==1.13.2 +tenacity==8.5.0 +threadpoolctl==3.5.0 +tiktoken==0.7.0 +tokenizers==0.19.1 +tomli==2.0.2 +torch==2.4.0 +torchvision==0.19.0 +tqdm==4.66.5 +transformers==4.44.2 +triton==3.0.0 +Twisted==24.7.0 +txaio==23.1.1 +txtorcon==24.8.0 +typer==0.12.4 +typing-inspect==0.9.0 +typing_extensions==4.12.2 +tzdata==2024.1 +urllib3==2.2.2 +uvicorn==0.30.6 +uvloop==0.20.0 +vllm==0.5.5 +vllm-flash-attn==2.6.1 +watchdog==4.0.2 +watchfiles==0.23.0 +wcwidth==0.2.13 +websockets==13.0 +xformers==0.0.27.post2 +xxhash==3.5.0 +yarl==1.9.4 +zipp==3.20.1 +zipstream-ng==1.7.1 +zope.interface==7.0.3