chore: add llama3.3

bentoml · Dec 27, 2024 · 2462ded · 2462ded
1 parent 66ac5d7
commit 2462ded
Showing 1 changed file with 18 additions and 0 deletions.
diff --git a/src/recipe.yaml b/src/recipe.yaml
@@ -1123,3 +1123,21 @@
   engine_config:
     model: Qwen/Qwen2.5-Coder-32B-Instruct
     max_model_len: 20480
+"llama3.3:70b-instruct-fp16":
+  project: vllm-chat
+  service_config:
+    name: llama3.3
+    traffic:
+      timeout: 300
+    resources:
+      gpu: 2
+      gpu_type: nvidia-a100-80gb
+  engine_config:
+    model: meta-llama/Llama-3.3-70B-Instruct
+    max_model_len: 2048
+    tensor_parallel_size: 2
+  extra_labels:
+    openllm_alias: 70b,70b-instruct
+    model_name: meta-llama/Llama-3.3-70B-Instruct
+  extra_envs:
+    - name: HF_TOKEN