Upgrade to QNN SDK 2.28 & update Python sample code

samples/python/README.md

@@ -55,5 +55,7 @@ python stable_diffusion_v2_1\stable_diffusion_v2_1.py --prompt "spectacular view
 | stable_diffusion_v1_5  | 2.24 | python stable_diffusion_v2_1\stable_diffusion_v2_1.py --prompt "the prompt string ..." |
 | stable_diffusion_v2_1  | 2.24 | python stable_diffusion_v1_5\stable_diffusion_v1_5.py --prompt "the prompt string ..." |
 | riffusion  | 2.24 | python riffusion\riffusion.py --prompt "the prompt string ..." |
+| real_esrgan_x4plus  | 2.28 | python real_esrgan_x4plus\real_esrgan_x4plus.py |
+| real_esrgan_general_x4v3  | 2.28 | python real_esrgan_general_x4v3\real_esrgan_general_x4v3.py |
 
 *More models will be supported soon!*

samples/python/aotgan/README.md

@@ -9,11 +9,11 @@ https://github.com/quic/ai-engine-direct-helper/blob/main/Docs/User_Guide.md
 
 Copy the QNN libraries from QNN SDK to below path:
 ```
-C:\ai-hub\aotgan\qnn\libQnnHtpV73Skel.so
-C:\ai-hub\aotgan\qnn\QnnHtp.dll
-C:\ai-hub\aotgan\qnn\QnnHtpV73Stub.dll
-C:\ai-hub\aotgan\qnn\QnnSystem.dll
-C:\ai-hub\aotgan\qnn\libqnnhtpv73.cat
+C:\ai-hub\aotgan\qai_libs\libQnnHtpV73Skel.so
+C:\ai-hub\aotgan\qai_libs\QnnHtp.dll
+C:\ai-hub\aotgan\qai_libs\QnnHtpV73Stub.dll
+C:\ai-hub\aotgan\qai_libs\QnnSystem.dll
+C:\ai-hub\aotgan\qai_libs\libqnnhtpv73.cat
 ```
 
 ## aotgan QNN models

samples/python/aotgan/aotgan.py

@@ -70,7 +70,7 @@ def Init():
     global aotgan
 
     # Config AppBuilder environment.
-    QNNConfig.Config(os.getcwd() + "\\qnn", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
+    QNNConfig.Config(os.getcwd() + "\\qai_libs", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
 
     # Instance for AotGan objects.
     aotgan_model = "models\\aotgan.bin"

samples/python/docs/guide.md

@@ -17,13 +17,11 @@ C:\ai-hub\model_name\
 
 Set up a new folder called `qnn` and copy the QNN libraries from QNN SDK to below path:
 ```
-C:\ai-hub\model_name\qnn\libqnnhtpv73.cat
-C:\ai-hub\model_name\qnn\libQnnHtpV73Skel.so
-C:\ai-hub\model_name\qnn\QnnCpu.dll
-C:\ai-hub\model_name\qnn\QnnHtp.dll
-C:\ai-hub\model_name\qnn\QnnHtpPrepare.dll
-C:\ai-hub\model_name\qnn\QnnHtpV73Stub.dll
-C:\ai-hub\model_name\qnn\QnnSystem.dll
+C:\ai-hub\model_name\qai_libs\libqnnhtpv73.cat
+C:\ai-hub\model_name\qai_libs\libQnnHtpV73Skel.so
+C:\ai-hub\model_name\qai_libs\QnnHtp.dll
+C:\ai-hub\model_name\qai_libs\QnnHtpV73Stub.dll
+C:\ai-hub\model_name\qai_libs\QnnSystem.dll
 ```
 
 ## Prepare the QNN model
@@ -174,7 +172,7 @@ def Init():
     global lamadilated
 
     # Config AppBuilder environment.
-    QNNConfig.Config(os.getcwd() + "\\qnn", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
+    QNNConfig.Config(os.getcwd() + "\\qai_libs", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
 
     # Instance for LamaDilated objects.
     lamadilated_model = "models\\lama_dilated.bin"

samples/python/fastsam_x/README.md

@@ -9,11 +9,11 @@ https://github.com/quic/ai-engine-direct-helper/blob/main/Docs/User_Guide.md
 
 Copy the QNN libraries from QNN SDK to below path:
 ```
-C:\ai-hub\unet_segmentation\qnn\libQnnHtpV73Skel.so
-C:\ai-hub\unet_segmentation\qnn\QnnHtp.dll
-C:\ai-hub\unet_segmentation\qnn\QnnHtpV73Stub.dll
-C:\ai-hub\unet_segmentation\qnn\QnnSystem.dll
-C:\ai-hub\unet_segmentation\qnn\libqnnhtpv73.cat
+C:\ai-hub\unet_segmentation\qai_libs\libQnnHtpV73Skel.so
+C:\ai-hub\unet_segmentation\qai_libs\QnnHtp.dll
+C:\ai-hub\unet_segmentation\qai_libs\QnnHtpV73Stub.dll
+C:\ai-hub\unet_segmentation\qai_libs\QnnSystem.dll
+C:\ai-hub\unet_segmentation\qai_libs\libqnnhtpv73.cat
 ```
 
 ## unet_segmentation QNN models

samples/python/fastsam_x/fastsam_x.py

@@ -151,7 +151,7 @@ def Init():
     global fastsam
 
     # Config AppBuilder environment.
-    QNNConfig.Config(os.getcwd() + "\\qnn", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
+    QNNConfig.Config(os.getcwd() + "\\qai_libs", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
 
     # Instance for FastSam_x objects.
     fastsam_model = "models\\fastsam_x.bin"

samples/python/inception_v3/README.md

@@ -10,11 +10,11 @@ https://github.com/quic/ai-engine-direct-helper/blob/main/Docs/User_Guide.md
 
 Copy the QNN libraries from QNN SDK to below path:
 ```
-C:\ai-hub\inception_v3\qnn\libQnnHtpV73Skel.so
-C:\ai-hub\inception_v3\qnn\QnnHtp.dll
-C:\ai-hub\inception_v3\qnn\QnnHtpV73Stub.dll
-C:\ai-hub\inception_v3\qnn\QnnSystem.dll
-C:\ai-hub\inception_v3\qnn\libqnnhtpv73.cat
+C:\ai-hub\inception_v3\qai_libs\libQnnHtpV73Skel.so
+C:\ai-hub\inception_v3\qai_libs\QnnHtp.dll
+C:\ai-hub\inception_v3\qai_libs\QnnHtpV73Stub.dll
+C:\ai-hub\inception_v3\qai_libs\QnnSystem.dll
+C:\ai-hub\inception_v3\qai_libs\libqnnhtpv73.cat
 ```
 
 ## inception_v3 QNN models

samples/python/inception_v3/inception_v3.py

@@ -50,7 +50,7 @@ def Init():
     global inceptionV3
 
     # Config AppBuilder environment.
-    QNNConfig.Config(os.getcwd() + "\\qnn", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
+    QNNConfig.Config(os.getcwd() + "\\qai_libs", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
 
     # Instance for InceptionV3 objects.
     inceptionV3_model = "models\\inception_v3.bin"

samples/python/lama_dilated/README.md

@@ -9,11 +9,11 @@ https://github.com/quic/ai-engine-direct-helper/blob/main/Docs/User_Guide.md
 
 Copy the QNN libraries from QNN SDK to below path:
 ```
-C:\ai-hub\lama_dilated\qnn\libQnnHtpV73Skel.so
-C:\ai-hub\lama_dilated\qnn\QnnHtp.dll
-C:\ai-hub\lama_dilated\qnn\QnnHtpV73Stub.dll
-C:\ai-hub\lama_dilated\qnn\QnnSystem.dll
-C:\ai-hub\lama_dilated\qnn\libqnnhtpv73.cat
+C:\ai-hub\lama_dilated\qai_libs\libQnnHtpV73Skel.so
+C:\ai-hub\lama_dilated\qai_libs\QnnHtp.dll
+C:\ai-hub\lama_dilated\qai_libs\QnnHtpV73Stub.dll
+C:\ai-hub\lama_dilated\qai_libs\QnnSystem.dll
+C:\ai-hub\lama_dilated\qai_libs\libqnnhtpv73.cat
 ```
 
 ## lama_dilated QNN models

samples/python/lama_dilated/lama_dilated.py

@@ -70,7 +70,7 @@ def Init():
     global lamadilated
 
     # Config AppBuilder environment.
-    QNNConfig.Config(os.getcwd() + "\\qnn", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
+    QNNConfig.Config(os.getcwd() + "\\qai_libs", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
 
     # Instance for LamaDilated objects.
     lamadilated_model = "models\\lama_dilated.bin"

samples/python/openpose/README.md

@@ -9,11 +9,11 @@ https://github.com/quic/ai-engine-direct-helper/blob/main/Docs/User_Guide.md
 
 Copy the QNN libraries from QNN SDK to below path:
 ```
-C:\ai-hub\openpose\qnn\libQnnHtpV73Skel.so
-C:\ai-hub\openpose\qnn\QnnHtp.dll
-C:\ai-hub\openpose\qnn\QnnHtpV73Stub.dll
-C:\ai-hub\openpose\qnn\QnnSystem.dll
-C:\ai-hub\openpose\qnn\libqnnhtpv73.cat
+C:\ai-hub\openpose\qai_libs\libQnnHtpV73Skel.so
+C:\ai-hub\openpose\qai_libs\QnnHtp.dll
+C:\ai-hub\openpose\qai_libs\QnnHtpV73Stub.dll
+C:\ai-hub\openpose\qai_libs\QnnSystem.dll
+C:\ai-hub\openpose\qai_libs\libqnnhtpv73.cat
 ```
 
 ## openpose QNN models

samples/python/openpose/openpose.py

@@ -427,7 +427,7 @@ def Init():
     global openpose
 
     # Config AppBuilder environment.
-    QNNConfig.Config(os.getcwd() + "\\qnn", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
+    QNNConfig.Config(os.getcwd() + "\\qai_libs", Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
 
     # Instance for OpnPose objects.
     openpose_model = "models\\openpose.bin"

samples/python/real_esrgan_general_x4v3/README.md

@@ -9,11 +9,11 @@ https://github.com/quic/ai-engine-direct-helper/blob/main/Docs/User_Guide.md
 
 Copy the QNN libraries from QNN SDK to below path:
 ```
-C:\ai-hub\real_esrgan_general_x4v3\qnn\libQnnHtpV73Skel.so
-C:\ai-hub\real_esrgan_general_x4v3\qnn\QnnHtp.dll
-C:\ai-hub\real_esrgan_general_x4v3\qnn\QnnHtpV73Stub.dll
-C:\ai-hub\real_esrgan_general_x4v3\qnn\QnnSystem.dll
-C:\ai-hub\real_esrgan_general_x4v3\qnn\libqnnhtpv73.cat
+C:\ai-hub\real_esrgan_general_x4v3\qai_libs\libQnnHtpV73Skel.so
+C:\ai-hub\real_esrgan_general_x4v3\qai_libs\QnnHtp.dll
+C:\ai-hub\real_esrgan_general_x4v3\qai_libs\QnnHtpV73Stub.dll
+C:\ai-hub\real_esrgan_general_x4v3\qai_libs\QnnSystem.dll
+C:\ai-hub\real_esrgan_general_x4v3\qai_libs\libqnnhtpv73.cat
 ```
 
 ## real_esrgan_general_x4v3 QNN models
@@ -25,10 +25,9 @@ You can refer to below links on how to setup AI Hub envirinment and how to use A
 https://aihub.qualcomm.com/get-started
 http://app.aihub.qualcomm.com/docs/
 
-a. Download the latest 'ai-hub-models' code and install it to Python environment:
+a. Install the AI Hub Python packages:
 ```
-git clone --recursive https://github.com/quic/ai-hub-models.git
-pip install -e .
+pip install qai-hub qai_hub_models
 ```
 
 b. Use below commmand to generate QNN model which suppor 515x512 input resolution:
@@ -45,7 +44,7 @@ Scheduled compile job (j1p86jxog) successfully. To see the status and results:
 ```
 After downloaded the model, copy it to the following path:
 ```
-C:\ai-hub\real_esrgan_general_x4v3\models\real_esrgan_general_x4v3_512.bin
+C:\ai-hub\real_esrgan_general_x4v3\models\real_esrgan_general_x4v3.bin
 ```
 
 ## Run the sample code

samples/python/real_esrgan_general_x4v3/real_esrgan_general_x4v3.py

@@ -3,10 +3,13 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # ---------------------------------------------------------------------
 
+import sys
 import os
+sys.path.append(".")
+sys.path.append("..")
+import utils.install as install
 import cv2
 import numpy as np
-
 import torch
 import torchvision.transforms as transforms
 from PIL import Image
@@ -16,17 +19,29 @@
 
 ####################################################################
 
+MODEL_ID = "mn0j68p8m"
+MODEL_NAME = "real_esrgan_general_x4v3"
+MODEL_HELP_URL = "https://github.com/quic/ai-engine-direct-helper/tree/main/samples/python/" + MODEL_NAME + "#" + MODEL_NAME + "-qnn-models"
+IMAGE_SIZE = 512
+
+####################################################################
+
 execution_ws = os.getcwd()
-qnn_dir = execution_ws + "\\qnn"
+qnn_dir = execution_ws + "\\qai_libs"
+
+if not MODEL_NAME in execution_ws:
+    execution_ws = execution_ws + "\\" + MODEL_NAME
+
+model_dir = execution_ws + "\\models"
+madel_path = model_dir + "\\" + MODEL_NAME + ".bin"
 
-image_size = 512
 image_buffer = None
 realesrgan = None
 
 def preprocess_PIL_image(image: Image) -> torch.Tensor:
     """Convert a PIL image into a pyTorch tensor with range [0, 1] and shape NCHW."""
-    transform = transforms.Compose([transforms.Resize(image_size),      # bgr image
-                                    transforms.CenterCrop(image_size),
+    transform = transforms.Compose([transforms.Resize(IMAGE_SIZE),      # bgr image
+                                    transforms.CenterCrop(IMAGE_SIZE),
                                     transforms.PILToTensor()])
     img: torch.Tensor = transform(image)  # type: ignore
     img = img.float() / 255.0  # int 0 - 255 to float 0.0 - 1.0
@@ -47,15 +62,26 @@ def Inference(self, input_data):
         output_data = super().Inference(input_datas)[0]        
         return output_data
 
+def model_download():
+    ret = True
+
+    desc = f"Downloading {MODEL_NAME} model... "
+    fail = f"\nFailed to download {MODEL_NAME} model. Please prepare the model according to the steps in below link:\n{MODEL_HELP_URL}"
+    ret = install.download_qai_hubmodel(MODEL_ID, madel_path, desc=desc, fail=fail)
+
+    if not ret:
+        exit()
+
 def Init():
     global realesrgan
 
+    model_download()
+
     # Config AppBuilder environment.
     QNNConfig.Config(qnn_dir, Runtime.HTP, LogLevel.WARN, ProfilingLevel.BASIC)
 
     # Instance for RealESRGan objects.
-    realesrgan_model = "models\\real_esrgan_general_x4v3_512.bin"
-    realesrgan = RealESRGan("realesrgan", realesrgan_model)
+    realesrgan = RealESRGan("realesrgan", madel_path)
 
 def Inference(input_image_path, output_image_path):
     global image_buffer
@@ -75,11 +101,12 @@ def Inference(input_image_path, output_image_path):
     PerfProfile.RelPerfProfileGlobal()
 
     output_image = torch.from_numpy(output_image)
-    output_image = output_image.reshape(image_size * 4, image_size * 4, 3)
+    output_image = output_image.reshape(IMAGE_SIZE * 4, IMAGE_SIZE * 4, 3)
     output_image = torch.unsqueeze(output_image, 0)
     output_image = [torch_tensor_to_PIL_image(img) for img in output_image]
     image_buffer = output_image[0]
     image_buffer.save(output_image_path)
+    image_buffer.show()
 
 def Release():
     global realesrgan
@@ -90,7 +117,7 @@ def Release():
 
 Init()
 
-Inference("input.png", "output.png")
+Inference(execution_ws + "\\input.png", execution_ws + "\\output.png")
 
 Release()
 

samples/python/real_esrgan_x4plus/README.md

@@ -9,11 +9,11 @@ https://github.com/quic/ai-engine-direct-helper/blob/main/Docs/User_Guide.md
 
 Copy the QNN libraries from QNN SDK to below path:
 ```
-C:\ai-hub\real_esrgan_x4plus\qnn\libQnnHtpV73Skel.so
-C:\ai-hub\real_esrgan_x4plus\qnn\QnnHtp.dll
-C:\ai-hub\real_esrgan_x4plus\qnn\QnnHtpV73Stub.dll
-C:\ai-hub\real_esrgan_x4plus\qnn\QnnSystem.dll
-C:\ai-hub\real_esrgan_x4plus\qnn\libqnnhtpv73.cat
+C:\ai-hub\real_esrgan_x4plus\qai_libs\libQnnHtpV73Skel.so
+C:\ai-hub\real_esrgan_x4plus\qai_libs\QnnHtp.dll
+C:\ai-hub\real_esrgan_x4plus\qai_libs\QnnHtpV73Stub.dll
+C:\ai-hub\real_esrgan_x4plus\qai_libs\QnnSystem.dll
+C:\ai-hub\real_esrgan_x4plus\qai_libs\libqnnhtpv73.cat
 ```
 
 ## real_esrgan_x4plus QNN models
@@ -25,10 +25,9 @@ You can refer to below links on how to setup AI Hub envirinment and how to use A
 https://aihub.qualcomm.com/get-started
 http://app.aihub.qualcomm.com/docs/
 
-a. Download the latest 'ai-hub-models' code and install it to Python environment:
+a. Install the AI Hub Python packages:
 ```
-git clone --recursive https://github.com/quic/ai-hub-models.git
-pip install -e .
+pip install qai-hub qai_hub_models
 ```
 
 b. Use below commmand to generate QNN model which suppor 515x512 input resolution:
@@ -45,7 +44,7 @@ Scheduled compile job (j1p86jxog) successfully. To see the status and results:
 ```
 After downloaded the model, copy it to the following path:
 ```
-C:\ai-hub\real_esrgan_x4plus\models\realesrgan_x4_512.bin
+C:\ai-hub\real_esrgan_x4plus\models\real_esrgan_x4plus.bin
 ```
 
 ## Run the sample code