02_tfidf_logreg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a28002f0-687a-49da-952e-d9f0c4b8967b",
   "metadata": {},
   "source": [
    "#### Логистическая регрессия (TFIDF+SVD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "281f89dd-71a0-47f2-a2d6-31bbeae72f73",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5365e96a-e42a-4a45-8d7c-84b4be22924e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "630914fd-d579-467a-a583-cb3dfac62462",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dcde5488-e415-4ade-8291-95bc3629afa6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "242ca1b0-3a5b-4d92-aafd-2f16305f0cc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_feather('data/prep.ftr')\n",
    "X, y = df.drop(columns='topic'), df['topic']\n",
    "params = {'stratify': y, 'test_size': 0.2, 'random_state': 42}\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, **params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afd948fe-6dcb-4ed4-b5a0-60bb2ee98a76",
   "metadata": {},
   "source": [
    "* *загружаем предобработанный датафрейм*\n",
    "* *разделяем на 2е части: для обучения и валидации*\n",
    "* *80% данных в обучении, стратификация по классам*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "358cf135-b334-4ef7-bfca-2e0c4e59fe22",
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg_params = {\n",
    "    'solver': 'liblinear'\n",
    "}\n",
    "\n",
    "svd_params = {\n",
    "    'n_iter': 5, \n",
    "    'random_state': 42, \n",
    "    'algorithm': 'randomized'\n",
    "}\n",
    "\n",
    "tfidf_params = {\n",
    "    'max_df': 0.8, \n",
    "    'min_df': 0.0004, \n",
    "    'dtype': np.float32, \n",
    "    'ngram_range': (1,2), \n",
    "    'token_pattern': None, \n",
    "    'tokenizer': str.split\n",
    "}\n",
    "\n",
    "sca = ('sca', StandardScaler())\n",
    "vec = ('vec', TfidfVectorizer(**tfidf_params))\n",
    "clf = ('clf', LogisticRegression(**logreg_params))\n",
    "\n",
    "text_svd = ('text_svd', TruncatedSVD(50, **svd_params))\n",
    "title_svd = ('title_svd', TruncatedSVD(300, **svd_params))\n",
    "\n",
    "text_vec = ('text_vec', Pipeline([vec, text_svd]), 'text')\n",
    "title_vec = ('title_vec', Pipeline([vec, title_svd]), 'title')\n",
    "trans = ('trans', ColumnTransformer([text_vec, title_vec], remainder='passthrough'))\n",
    "\n",
    "logreg = Pipeline([trans, sca, clf])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4faa3399-b494-4100-89b9-52d420597d02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(345726, 35333)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TfidfVectorizer(**tfidf_params).fit_transform(X_train['text']).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f076644f-41cb-4f20-803e-a12646dcee70",
   "metadata": {},
   "source": [
    "* *собираем pipeline: TFIDF -> SVD -> Scaler -> LogReg*\n",
    "* *независимая векторизация title и text (50 и 300 фичей соответственно)*\n",
    "* *TFIDF: униграммы и биграммы, после подбора min_df и max_df получаем 35333 токенов в словаре*\n",
    "* *на входе также будут title_len и text_len из предобработанного датафрейма (тоже попадут в Scaler)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47a3d27c-965d-4118-9a7d-0172699aa835",
   "metadata": {},
   "outputs": [],
   "source": [
    "%time logreg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a15d62d-dff2-4626-a688-6c260e316908",
   "metadata": {},
   "source": [
    "* *обучение модели на 80% данных занимает ~15мин*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e38b3dde-6f70-4588-be1a-d55845a78d9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 28.3 s, sys: 456 ms, total: 28.8 s\n",
      "Wall time: 28.8 s\n"
     ]
    }
   ],
   "source": [
    "%time y_pred = logreg.predict(X_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fe396d6a-77a1-40ca-ac2b-1ff54f0d4a81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9043409848204369"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_valid, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c453bb35-e784-4408-855b-c157b6d439ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8939069567118967"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_valid, y_pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "524df81d-f526-426a-8ef1-1bde8cfa1fe7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 6698   377   805   447   101   505]\n",
      " [  288  9708   522   103    33   105]\n",
      " [  387   298 25461   531   108   539]\n",
      " [  384   160   641  9130    23   289]\n",
      " [   58    27   160    30 12569    39]\n",
      " [  330    74   662   177    65 14598]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_valid, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8a5b0bb2-80e9-400d-89c8-7ef3da86e432",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.75      0.78      8933\n",
      "           1       0.91      0.90      0.91     10759\n",
      "           2       0.90      0.93      0.92     27324\n",
      "           3       0.88      0.86      0.87     10627\n",
      "           4       0.97      0.98      0.98     12883\n",
      "           5       0.91      0.92      0.91     15906\n",
      "\n",
      "    accuracy                           0.90     86432\n",
      "   macro avg       0.90      0.89      0.89     86432\n",
      "weighted avg       0.90      0.90      0.90     86432\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_valid, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b0b65243-c56c-474a-970f-23b73e05a587",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Интернет и СМИ', 'Культура', 'Мир', 'Наука и техника', 'Спорт', 'Экономика']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(load_json('data/labels.json').keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cb2b1d1-b820-442c-9a49-fd48bf9efaab",
   "metadata": {},
   "source": [
    "* *наименее точно модель классифицирует класс 0 (Интернет и СМИ)*\n",
    "* *данный класс имеет наименьшее количество объектов в обучающей выборке (и в датасете)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "319271c2-9d9a-459f-9df4-633e458dfce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "dump_pickle(logreg, 'data/logreg.bin')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ea8e24e-32bc-47da-be24-eabc4acb8b24",
   "metadata": {},
   "source": [
    "* *данная модель занимает достаточно много места на диске и в памяти (файл модели ~400Мб)*\n",
    "* *в данном случае внутри pipeline имеем обученный TFIDF и SVD, отсюда и такой размер модели*\n",
    "* *кроме того, TFIDF и SVD внутри модели скорее всего приведут к более низкой производительности*"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dev",
   "language": "python",
   "name": "dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}