From ed11740c17cce40fa553bd9cf30ad052bed7f544 Mon Sep 17 00:00:00 2001
From: Anokhov <artyom.anokhov@intel.com>
Date: Mon, 26 Aug 2024 13:10:53 +0200
Subject: [PATCH] Initial commit

---
 templates/rhel8/dist/dev.dockerfile.j2           | 12 +++++++++---
 templates/rhel8/env/dev_env.dockerfile.j2        |  1 +
 templates/rhel8/env/runtime_env.dockerfile.j2    |  1 +
 templates/ubuntu20/dist/dev.dockerfile.j2        | 12 ++++++++----
 templates/ubuntu20/env/dev_env.dockerfile.j2     |  1 +
 templates/ubuntu20/env/runtime_env.dockerfile.j2 |  1 +
 templates/ubuntu22/dist/dev.dockerfile.j2        | 12 ++++++++----
 templates/ubuntu22/env/dev_env.dockerfile.j2     |  1 +
 templates/ubuntu22/env/runtime_env.dockerfile.j2 |  1 +
 9 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/templates/rhel8/dist/dev.dockerfile.j2 b/templates/rhel8/dist/dev.dockerfile.j2
index 695f6898..8b172509 100644
--- a/templates/rhel8/dist/dev.dockerfile.j2
+++ b/templates/rhel8/dist/dev.dockerfile.j2
@@ -5,10 +5,16 @@ ARG OPENVINO_WHEELS_URL
 # hadolint ignore=SC2102,DL3033
 RUN yum install -y cmake git && yum clean all && \
     if [ -z "$OPENVINO_WHEELS_URL" ]; then \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}.0" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-genai=="${OPENVINO_WHEELS_VERSION}.0" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --extra-index-url https://download.pytorch.org/whl/cpu; \
     else \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu ; \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}.0" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-genai=="${OPENVINO_WHEELS_VERSION}.0" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu ; \
     fi
 
 # download source for PyPi LGPL packages
diff --git a/templates/rhel8/env/dev_env.dockerfile.j2 b/templates/rhel8/env/dev_env.dockerfile.j2
index dcf972ec..481a0712 100644
--- a/templates/rhel8/env/dev_env.dockerfile.j2
+++ b/templates/rhel8/env/dev_env.dockerfile.j2
@@ -8,4 +8,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig
diff --git a/templates/rhel8/env/runtime_env.dockerfile.j2 b/templates/rhel8/env/runtime_env.dockerfile.j2
index 9eb25ba5..476d274d 100644
--- a/templates/rhel8/env/runtime_env.dockerfile.j2
+++ b/templates/rhel8/env/runtime_env.dockerfile.j2
@@ -8,4 +8,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig
diff --git a/templates/ubuntu20/dist/dev.dockerfile.j2 b/templates/ubuntu20/dist/dev.dockerfile.j2
index e4983e2b..f7a1a1a3 100644
--- a/templates/ubuntu20/dist/dev.dockerfile.j2
+++ b/templates/ubuntu20/dist/dev.dockerfile.j2
@@ -5,11 +5,15 @@ ARG OPENVINO_WHEELS_URL
 # hadolint ignore=SC2102
 RUN apt-get update && apt-get install -y --no-install-recommends cmake make git && rm -rf /var/lib/apt/lists/* && \
     if [ -z "$OPENVINO_WHEELS_URL" ]; then \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="$OPENVINO_WHEELS_VERSION" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}.0" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-genai=="${OPENVINO_WHEELS_VERSION}.0" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --extra-index-url https://download.pytorch.org/whl/cpu; \
     else \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}.0" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-genai=="${OPENVINO_WHEELS_VERSION}.0" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu; \
     fi
 
 WORKDIR ${INTEL_OPENVINO_DIR}/licensing
diff --git a/templates/ubuntu20/env/dev_env.dockerfile.j2 b/templates/ubuntu20/env/dev_env.dockerfile.j2
index dcf972ec..481a0712 100644
--- a/templates/ubuntu20/env/dev_env.dockerfile.j2
+++ b/templates/ubuntu20/env/dev_env.dockerfile.j2
@@ -8,4 +8,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig
diff --git a/templates/ubuntu20/env/runtime_env.dockerfile.j2 b/templates/ubuntu20/env/runtime_env.dockerfile.j2
index 9eb25ba5..476d274d 100644
--- a/templates/ubuntu20/env/runtime_env.dockerfile.j2
+++ b/templates/ubuntu20/env/runtime_env.dockerfile.j2
@@ -8,4 +8,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig
diff --git a/templates/ubuntu22/dist/dev.dockerfile.j2 b/templates/ubuntu22/dist/dev.dockerfile.j2
index 41862a4b..99f22a0a 100644
--- a/templates/ubuntu22/dist/dev.dockerfile.j2
+++ b/templates/ubuntu22/dist/dev.dockerfile.j2
@@ -5,11 +5,15 @@ ARG OPENVINO_WHEELS_URL
 # hadolint ignore=SC2102
 RUN apt-get update && apt-get install -y --no-install-recommends cmake make git && rm -rf /var/lib/apt/lists/* && \
     if [ -z "$OPENVINO_WHEELS_URL" ]; then \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="$OPENVINO_WHEELS_VERSION" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}.0" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-genai=="${OPENVINO_WHEELS_VERSION}.0" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --extra-index-url https://download.pytorch.org/whl/cpu; \
     else \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}.0" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-genai=="${OPENVINO_WHEELS_VERSION}.0" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu; \
     fi
 
 WORKDIR ${INTEL_OPENVINO_DIR}/licensing
diff --git a/templates/ubuntu22/env/dev_env.dockerfile.j2 b/templates/ubuntu22/env/dev_env.dockerfile.j2
index 3b28b8cc..3c29bc5a 100644
--- a/templates/ubuntu22/env/dev_env.dockerfile.j2
+++ b/templates/ubuntu22/env/dev_env.dockerfile.j2
@@ -7,4 +7,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig
diff --git a/templates/ubuntu22/env/runtime_env.dockerfile.j2 b/templates/ubuntu22/env/runtime_env.dockerfile.j2
index 0fc5daba..8f6f8a28 100644
--- a/templates/ubuntu22/env/runtime_env.dockerfile.j2
+++ b/templates/ubuntu22/env/runtime_env.dockerfile.j2
@@ -7,4 +7,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig