使用python部署机器学习模型的10个实践经验.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://mp.weixin.qq.com/s?__biz=MjM5ODkzMzMwMQ==&mid=2650414738&idx=4&sn=291b3cd0a1abcbcbf5661fe9b5c4caed&chksm=becd98c889ba11deb392729961fbf844ed70bde1c203570f93b969e1f49c03f9a2672c795e76&scene=126&sessionid=1593691947&key=d28e5ef1d5d516753b90b3da896b87cc42d273354f534163bbbb745c2df4507e929838e9d27a6f41654879cd9ce6e89af580f2c02553edd65162ca0d33b184334300d9dec75d022ce6b4f3a890d96153&ascene=1&uin=MjA1MjAyODkxNg%3D%3D&devicetype=Windows+10+x64&version=6209051a&lang=zh_CN&exportkey=AY613ediuml%2BOiLzM4tOow8%3D&pass_ticket=TTbfflTude6XHy0P68%2Fx35vP%2Fnt7hPuBsvpukmQN97jygfZCqOotQywB77daKGb8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.学会写抽象类\n",
    "import os\n",
    "from abc import ABCMeta,abstractmethod\n",
    "class DataProcessor(metaclass=ABCMeta):\n",
    "    \"\"\"Base processor to be used for all preparation.\"\"\"\n",
    "    def __init__(self,input_directory,output_directory):\n",
    "        self.input_directory = input_directory\n",
    "        self.output_directory = output_directory\n",
    "    @abstractmethod\n",
    "    def read(self):\n",
    "        \"\"\"Read raw data.\"\"\"\n",
    "    @abstractmethod\n",
    "    def process(self):\n",
    "        \"\"\"Processes raw data.This step should create the raw dataframe with all the required features. Shouldn't implement statistical or text cleaning.\"\"\"\n",
    "    @abstractmethod\n",
    "    def save(self):\n",
    "        \"\"\"Saves processed data.\"\"\"\n",
    "class Trainer(metaclass=ABCMeta):\n",
    "    def __init__(self,directory):\n",
    "        self.directory = directory\n",
    "        self.model_directory = os.path.join(directory,'models')\n",
    "    @abstractmethod\n",
    "    def preprocess(self):\n",
    "        \"\"\"This takes the preprocessed data and returns clean data. This is more about statistical or text cleaning.\"\"\"\n",
    "    @abstractmethod\n",
    "    def set_model(self):\n",
    "        \"\"\"Define model here.\"\"\"\n",
    "    @abstractmethod\n",
    "    def fit_model(self):\n",
    "        \"\"\"This takes the vectorised data and returns a trained model.\"\"\"\n",
    "    @abstractmethod\n",
    "    def generate_metrics(self):\n",
    "        \"\"\"Generates metric with trained model and test data.\"\"\"\n",
    "    @abstractmethod\n",
    "    def save_model(self,model_name):\n",
    "        \"\"\"This method saves the model in our required format.\"\"\"\n",
    "class Predict(metaclass=ABCMeta):\n",
    "    \"\"\"Base predictor to be used for all models.\"\"\"\n",
    "    def __init__(self, directory):\n",
    "        self.directory = directory\n",
    "        self.model_directory = os.path.join(directory, 'models')\n",
    "    @abstractmethod\n",
    "    def load_model(self):\n",
    "        \"\"\"Load model here.\"\"\"\n",
    "    @abstractmethod\n",
    "    def preprocess(self):\n",
    "        \"\"\"This takes the raw data and returns clean data for prediction.\"\"\"\n",
    "    @abstractmethod\n",
    "    def predict(self):\n",
    "        \"\"\"This is used for prediction.\"\"\"\n",
    "class BaseDB(metaclass=ABCMeta):\n",
    "    \"\"\" Base database class to be used for all DB connectors.\"\"\"\n",
    "    def get_connection(self):\n",
    "        \"\"\"This creates a new DB connection.\"\"\"\n",
    "    def close_connection(self):\n",
    "        \"\"\"This closes the DB connection.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2.在最前面设置你的随机数种子\n",
    "def set_seed(args):\n",
    "    random.seed(args.seed)\n",
    "    np.random.seed(args.seed)\n",
    "    torch.manual_seed(args.seed)\n",
    "    if args.n_gpu > 0:\n",
    "        torch.cuda.manual_seed_all(args.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3.从几行数据开始\n",
    "df_train = pd.read_csv('train.csv',nrows=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4.预见失败（成熟开发人员的标志）\n",
    "print(len(df))\n",
    "df.isna().sum()\n",
    "df.dropna()\n",
    "print(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5.显示处理进度\n",
    "# 选项1-tqdm\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "tqdm.pandas()\n",
    "\n",
    "df['col'] = df['col'].progress_apply(lambda x: x**2)\n",
    "\n",
    "text = \"\"\n",
    "for char in tqdm([\"a\", \"b\", \"c\", \"d\"]):\n",
    "    time.sleep(0.25)\n",
    "    text = text + char\n",
    "# 选项2-fastprogress\n",
    "from fastprogress.fastprogress import master_bar, progress_bar\n",
    "from time import sleep\n",
    "mb = master_bar(range(10))\n",
    "for i in mb:\n",
    "    for j in progress_bar(range(100), parent=mb):\n",
    "        sleep(0.01)\n",
    "        mb.child.comment = f'second bar stat'\n",
    "    mb.first_bar.comment = f'first bar stat'\n",
    "    mb.write(f'Finished loop {i}.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6.Pandas很慢\n",
    "import modin.pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 7.统计函数的时间\n",
    "import time\n",
    "def timing(f):\n",
    "    \"\"\"Decorator for timing functions\n",
    "    Usage:\n",
    "    @timing\n",
    "    def function(a):\n",
    "        pass\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 8.不要再云上烧钱\n",
    "import os\n",
    "def run_command(cmd):\n",
    "    return os.system(cmd)\n",
    "def shutdown(seconds=0,os='linux'):\n",
    "    \"\"\"Shutdown system after seconds given. Useful for shutting EC2 to save costs.\"\"\"\n",
    "    if os == 'linux':\n",
    "        run_command('sudo shutdown -h -t sec %s'%seconds)\n",
    "    elif os == 'windows':\n",
    "        run_command('shutdown -s -t %s'%seconds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建和保存报告\n",
    "import json\n",
    "import os\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, f1_score, fbeta_score\n",
    "\n",
    "\n",
    "def get_metrics(y, y_pred, beta=2, average_method='macro', y_encoder=None):\n",
    "    if y_encoder:\n",
    "        y = y_encoder.inverse_transform(y)\n",
    "        y_pred = y_encoder.inverse_transform(y_pred)\n",
    "    return {\n",
    "        'accuracy': round(accuracy_score(y, y_pred), 4),\n",
    "        'f1_score_macro': round(f1_score(y, y_pred, average=average_method), 4),\n",
    "        'fbeta_score_macro': round(fbeta_score(y, y_pred, beta, average=average_method), 4),\n",
    "        'report': classification_report(y, y_pred, output_dict=True),\n",
    "        'report_csv': classification_report(y, y_pred, output_dict=False).replace('\\n', '\\r\\n')\n",
    "    }\n",
    "\n",
    "\n",
    "def save_metrics(metrics: dict, model_directory, file_name):\n",
    "    path = os.path.join(model_directory, file_name + '_report.txt')\n",
    "    classification_report_to_csv(metrics['report_csv'], path)\n",
    "    metrics.pop('report_csv')\n",
    "    path = os.path.join(model_directory, file_name + '_metrics.json')\n",
    "    json.dump(metrics, open(path, 'w'), indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 写好APIs\n",
    "下面是在不太高的负载下(比如1000/min)部署经典的ML和DL的好方法。\n",
    "fasbut + uvicorn\n",
    "Fastest — 使用fastapi编写API，因为它很快。\n",
    "Documentation — 用fastapi写API让我们不用操心文档。\n",
    "Workers — 使用uvicorn部署API"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}