Merge branch 'main' of gitlab.tugraz.at:julik/air-project

1c55ed8b · Kienzl, Julian · 57db0244 · c4db4339 · 1c55ed8b
Commit 1c55ed8b authored Jan 7, 2024 by Kienzl, Julian
--- a/main.ipynb
+++ b/main.ipynb
@@ -128,7 +128,7 @@
   "source": [
    "dataset = pd.read_csv(\n",
    "    \"data/train_short_200000.csv\", names=[\"sentiment\", \"title\", \"text\"], sep=\",\"\n",
-    ").head(1000)\n",
+    ").head(10000)\n",
    "dataset.text = dataset.title + \" \" + dataset.text\n",
    "\n",
    "dataset = dataset.dropna()\n",
@@ -252,7 +252,19 @@
   "metadata": {},
   "source": [
    "### Sentiment Analysis with BERT \n",
-    "TODO: Description"
+    "Initialization\n",
+    "-We create a sentiment-analysis pipeline with our pretrained model bert_sentiment_model.\n",
+    "  \n",
+    "  \n",
+    "To determine the sentiment of a given query and of the most similar reviews to that query we have to function:\n",
+    "- get_sentiment_of_query for the query  \n",
+    "- get_sentiment_for_each_result for the similar reviews\n",
+    "\n",
+    "In get_sentiment_of_query we return a tuple that contains the sentiment and the sentiment score. Both of those values are returned by the sentiment-analysis pipeline.  \n",
+    "In get_sentiment_for_each_result we return a pd.dataframe that added both values(sentiment and the sentiment score) to a given pd.dataframe. For each of these values a new column is created within the pd.dataframe.\n",
+    "    \n",
+    "The higher the sentiment score is, the higher a text tends towards a sentiment. \n",
+    "  "
   ]
  },
  {
@@ -327,19 +339,40 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
-     "ename": "AttributeError",
+     "ename": "KeyboardInterrupt",
-     "evalue": "'Series' object has no attribute 'title'",
+     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
-      "\u001b[0;32m/tmp/ipykernel_4859/1683762815.py\u001b[0m in \u001b[0;36m?\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0mnumber_of_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m50\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0mpercentages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquery\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtest_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterrows\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     \u001b[0mquery_sentiment\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbert_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_sentiment_of_query\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m     query_results = bert_model.retrieve_top_k_entries_for_query(\n\u001b[1;32m     23\u001b[0m         \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnumber_of_results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m     )\n",
+      "Cell \u001b[0;32mIn[27], line 25\u001b[0m\n\u001b[1;32m     21\u001b[0m query_sentiment \u001b[38;5;241m=\u001b[39m bert_model\u001b[38;5;241m.\u001b[39mget_sentiment_of_query(query\u001b[38;5;241m=\u001b[39mquery\u001b[38;5;241m.\u001b[39mtext)\n\u001b[1;32m     22\u001b[0m query_results \u001b[38;5;241m=\u001b[39m bert_model\u001b[38;5;241m.\u001b[39mretrieve_top_k_entries_for_query(\n\u001b[1;32m     23\u001b[0m     query\u001b[38;5;241m=\u001b[39mquery\u001b[38;5;241m.\u001b[39mtext, k\u001b[38;5;241m=\u001b[39mnumber_of_results\n\u001b[1;32m     24\u001b[0m )\n\u001b[0;32m---> 25\u001b[0m query_results_with_sentiment \u001b[38;5;241m=\u001b[39m \u001b[43mbert_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_sentiment_for_each_result\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     26\u001b[0m \u001b[43m    \u001b[49m\u001b[43mresults\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquery_results\u001b[49m\n\u001b[1;32m     27\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     29\u001b[0m sentiment_distribution \u001b[38;5;241m=\u001b[39m Counter(query_results_with_sentiment\u001b[38;5;241m.\u001b[39msentiment\u001b[38;5;241m.\u001b[39mtolist())\n\u001b[1;32m     30\u001b[0m percentage \u001b[38;5;241m=\u001b[39m (sentiment_distribution[query_sentiment[\u001b[38;5;241m0\u001b[39m]] \u001b[38;5;241m/\u001b[39m number_of_results) \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m100\u001b[39m\n",
-      "\u001b[0;32m~/.local/lib/python3.11/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m?\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   6200\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_accessors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6201\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6202\u001b[0m         ):\n\u001b[1;32m   6203\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6204\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "Cell \u001b[0;32mIn[15], line 33\u001b[0m, in \u001b[0;36mBERT.get_sentiment_for_each_result\u001b[0;34m(self, results)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_sentiment_for_each_result\u001b[39m(\u001b[38;5;28mself\u001b[39m, results: pd\u001b[38;5;241m.\u001b[39mDataFrame) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame:\n\u001b[1;32m     32\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m index, row \u001b[38;5;129;01min\u001b[39;00m results\u001b[38;5;241m.\u001b[39miterrows():\n\u001b[0;32m---> 33\u001b[0m         r \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msentiment_analyzer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrow\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     34\u001b[0m         results\u001b[38;5;241m.\u001b[39mloc[index, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msentiment\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m r[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlabel\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m     35\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m results\n",
-      "\u001b[0;31mAttributeError\u001b[0m: 'Series' object has no attribute 'title'"
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/pipelines/text_classification.py:156\u001b[0m, in \u001b[0;36mTextClassificationPipeline.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    122\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m    123\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    124\u001b[0m \u001b[38;5;124;03m    Classify the text(s) given as inputs.\u001b[39;00m\n\u001b[1;32m    125\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    154\u001b[0m \u001b[38;5;124;03m        If `top_k` is used, one such dictionary is returned per label.\u001b[39;00m\n\u001b[1;32m    155\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 156\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    157\u001b[0m     \u001b[38;5;66;03m# TODO try and retrieve it in a nicer way from _sanitize_parameters.\u001b[39;00m\n\u001b[1;32m    158\u001b[0m     _legacy \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtop_k\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m kwargs\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/pipelines/base.py:1140\u001b[0m, in \u001b[0;36mPipeline.__call__\u001b[0;34m(self, inputs, num_workers, batch_size, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1132\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mnext\u001b[39m(\n\u001b[1;32m   1133\u001b[0m         \u001b[38;5;28miter\u001b[39m(\n\u001b[1;32m   1134\u001b[0m             \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_iterator(\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1137\u001b[0m         )\n\u001b[1;32m   1138\u001b[0m     )\n\u001b[1;32m   1139\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1140\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_single\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpreprocess_params\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mforward_params\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpostprocess_params\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/pipelines/base.py:1147\u001b[0m, in \u001b[0;36mPipeline.run_single\u001b[0;34m(self, inputs, preprocess_params, forward_params, postprocess_params)\u001b[0m\n\u001b[1;32m   1145\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrun_single\u001b[39m(\u001b[38;5;28mself\u001b[39m, inputs, preprocess_params, forward_params, postprocess_params):\n\u001b[1;32m   1146\u001b[0m     model_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpreprocess(inputs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mpreprocess_params)\n\u001b[0;32m-> 1147\u001b[0m     model_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mforward_params\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1148\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpostprocess(model_outputs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mpostprocess_params)\n\u001b[1;32m   1149\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m outputs\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/pipelines/base.py:1046\u001b[0m, in \u001b[0;36mPipeline.forward\u001b[0;34m(self, model_inputs, **forward_params)\u001b[0m\n\u001b[1;32m   1044\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m inference_context():\n\u001b[1;32m   1045\u001b[0m         model_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_ensure_tensor_on_device(model_inputs, device\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[0;32m-> 1046\u001b[0m         model_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mforward_params\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1047\u001b[0m         model_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_ensure_tensor_on_device(model_outputs, device\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mdevice(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n\u001b[1;32m   1048\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/pipelines/text_classification.py:187\u001b[0m, in \u001b[0;36mTextClassificationPipeline._forward\u001b[0;34m(self, model_inputs)\u001b[0m\n\u001b[1;32m    185\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124muse_cache\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m inspect\u001b[38;5;241m.\u001b[39msignature(model_forward)\u001b[38;5;241m.\u001b[39mparameters\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m    186\u001b[0m     model_inputs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124muse_cache\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 187\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/models/distilbert/modeling_distilbert.py:1000\u001b[0m, in \u001b[0;36mDistilBertForSequenceClassification.forward\u001b[0;34m(self, input_ids, attention_mask, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    992\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    993\u001b[0m \u001b[38;5;124;03mlabels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):\u001b[39;00m\n\u001b[1;32m    994\u001b[0m \u001b[38;5;124;03m    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,\u001b[39;00m\n\u001b[1;32m    995\u001b[0m \u001b[38;5;124;03m    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If\u001b[39;00m\n\u001b[1;32m    996\u001b[0m \u001b[38;5;124;03m    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).\u001b[39;00m\n\u001b[1;32m    997\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    998\u001b[0m return_dict \u001b[38;5;241m=\u001b[39m return_dict \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_return_dict\n\u001b[0;32m-> 1000\u001b[0m distilbert_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdistilbert\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1001\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1002\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1003\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1004\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1005\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1006\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1007\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1008\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1009\u001b[0m hidden_state \u001b[38;5;241m=\u001b[39m distilbert_output[\u001b[38;5;241m0\u001b[39m]  \u001b[38;5;66;03m# (bs, seq_len, dim)\u001b[39;00m\n\u001b[1;32m   1010\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m hidden_state[:, \u001b[38;5;241m0\u001b[39m]  \u001b[38;5;66;03m# (bs, dim)\u001b[39;00m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/models/distilbert/modeling_distilbert.py:820\u001b[0m, in \u001b[0;36mDistilBertModel.forward\u001b[0;34m(self, input_ids, attention_mask, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    817\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m attention_mask \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    818\u001b[0m         attention_mask \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mones(input_shape, device\u001b[38;5;241m=\u001b[39mdevice)  \u001b[38;5;66;03m# (bs, seq_length)\u001b[39;00m\n\u001b[0;32m--> 820\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtransformer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    821\u001b[0m \u001b[43m    \u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43membeddings\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    822\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattn_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    823\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    824\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    825\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    826\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    827\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/models/distilbert/modeling_distilbert.py:585\u001b[0m, in \u001b[0;36mTransformer.forward\u001b[0;34m(self, x, attn_mask, head_mask, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    577\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m    578\u001b[0m         layer_module\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[1;32m    579\u001b[0m         hidden_state,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    582\u001b[0m         output_attentions,\n\u001b[1;32m    583\u001b[0m     )\n\u001b[1;32m    584\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 585\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mlayer_module\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    586\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_state\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    587\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattn_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    588\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    589\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    590\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    592\u001b[0m hidden_state \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]\n\u001b[1;32m    594\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m output_attentions:\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/models/distilbert/modeling_distilbert.py:530\u001b[0m, in \u001b[0;36mTransformerBlock.forward\u001b[0;34m(self, x, attn_mask, head_mask, output_attentions)\u001b[0m\n\u001b[1;32m    528\u001b[0m \u001b[38;5;66;03m# Feed Forward Network\u001b[39;00m\n\u001b[1;32m    529\u001b[0m ffn_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mffn(sa_output)  \u001b[38;5;66;03m# (bs, seq_length, dim)\u001b[39;00m\n\u001b[0;32m--> 530\u001b[0m ffn_output: torch\u001b[38;5;241m.\u001b[39mTensor \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput_layer_norm\u001b[49m\u001b[43m(\u001b[49m\u001b[43mffn_output\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43msa_output\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# (bs, seq_length, dim)\u001b[39;00m\n\u001b[1;32m    532\u001b[0m output \u001b[38;5;241m=\u001b[39m (ffn_output,)\n\u001b[1;32m    533\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m output_attentions:\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/modules/normalization.py:196\u001b[0m, in \u001b[0;36mLayerNorm.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 196\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlayer_norm\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    197\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnormalized_shape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meps\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/torch/nn/functional.py:2543\u001b[0m, in \u001b[0;36mlayer_norm\u001b[0;34m(input, normalized_shape, weight, bias, eps)\u001b[0m\n\u001b[1;32m   2539\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_variadic(\u001b[38;5;28minput\u001b[39m, weight, bias):\n\u001b[1;32m   2540\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m   2541\u001b[0m         layer_norm, (\u001b[38;5;28minput\u001b[39m, weight, bias), \u001b[38;5;28minput\u001b[39m, normalized_shape, weight\u001b[38;5;241m=\u001b[39mweight, bias\u001b[38;5;241m=\u001b[39mbias, eps\u001b[38;5;241m=\u001b[39meps\n\u001b[1;32m   2542\u001b[0m     )\n\u001b[0;32m-> 2543\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlayer_norm\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnormalized_shape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meps\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackends\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcudnn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menabled\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],

 %% Cell type:markdown id: tags:
 # Semantic Search with Sentiment Comparison for a Given Query
 %% Cell type:markdown id: tags:
 **Group 29**
 | Name               | Matriculation Number |
 |--------------------|----------------------|
 | Benjamin Jost      | 11912846             |
 | Julian Kienzel     |                      |
 | Fabio Maierbrugger | 11908625             |
 %% Cell type:code id: tags:
 ``` python
 import json
 import textwrap
 import torch
 import warnings
 warnings.simplefilter(action='ignore', category=FutureWarning)
 import pandas as pd
 pd.options.mode.chained_assignment = None
 import matplotlib.pyplot as plt
 from collections import Counter
 from sentence_transformers import SentenceTransformer, util
 from transformers import pipeline, logging
 logging.set_verbosity_warning()
 ```
 %% Cell type:markdown id: tags:
 ### Preprocessing
 Here the already shortened [train-dataset](https://www.kaggle.com/datasets/kritanjalijain/amazon-reviews/data) (from 34.686.770 to 200.000 rows) is preprocessed by  the following steps:
 - Concatinating the columns title and text
 - Dropping null values
 - Dropping the title column
 - Dropping the sentiment column
 - Convert everything into lowercase
 %% Cell type:code id: tags:
 ``` python
 dataset = pd.read_csv(
    "data/train_short_200000.csv", names=["sentiment", "title", "text"], sep=","
-).head(1000)
+).head(10000)
 dataset.text = dataset.title + " " + dataset.text
 dataset = dataset.dropna()
 dataset = dataset.drop(["title"], axis=1)
 dataset = dataset.drop(["sentiment"], axis=1)
 dataset.text = dataset.text.str.lower()
 dataset.head()
 ```
 %% Output
                                                    text
    0  stuning even for the non-gamer this sound trac...
    1  the best soundtrack ever to anything. i'm read...
    2  amazing! this soundtrack is my favorite music ...
    3  excellent soundtrack i truly like this soundtr...
    4  remember, pull your jaw off the floor after he...
 %% Cell type:markdown id: tags:
 ### BERT class model
 %% Cell type:code id: tags:
 ``` python
 bert_sentence_model = "all-MiniLM-L12-v2"  # https://www.sbert.net/docs/pretrained_models.html
 bert_sentiment_model = "distilbert-base-uncased-finetuned-sst-2-english" # https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english
 class BERT:
    def __init__(self, dataset: pd.DataFrame) -> None:
        self.dataset = dataset
        self.sentence_transformer = SentenceTransformer(
            bert_sentence_model, device="cpu"
        )
        self.sentiment_analyzer = pipeline(
            "sentiment-analysis", model=bert_sentiment_model
        )
        torch.cuda.empty_cache()
        torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.content_embeddings = self.sentence_transformer.encode(
            self.dataset.text.tolist(), convert_to_tensor=True
        )
    def retrieve_top_k_entries_for_query(self, query: str, k: int):
        query_embedding = self.sentence_transformer.encode(
            query, convert_to_tensor=True
        )
        similarities = util.pytorch_cos_sim(query_embedding, self.content_embeddings)[0]
        top_k_indices = similarities.argsort(descending=True)[:k].cpu().numpy()
        return self.dataset.iloc[top_k_indices]
    def get_sentiment_for_each_result(self, results: pd.DataFrame) -> pd.DataFrame:
        for index, row in results.iterrows():
            r = self.sentiment_analyzer(row.text)
            results.loc[index, "sentiment"] = r[0]["label"]
        return results
    def get_sentiment_of_query(self, query: str) -> tuple:
        r = self.sentiment_analyzer(query)
        return r[0]["label"], r[0]["score"]
 ```
 %% Cell type:code id: tags:
 ``` python
 bert_model = BERT(dataset=dataset)
 # This takes a while...
 ```
 %% Cell type:markdown id: tags:
 ### Semantic Search with BERT
 Initialization
 - First our given dataset texts are converted to lists
 - The lists are then transformed into embeddings
 - Those embeddings are converted into a tensors
 For the query the same steps are applied in the beginning in retrieve_top_k_entries_for_query.
 Afterwards we calculate the most similar documents with the pytorch cosine similarity.
 We reorder the list to have the highest similarity values on top and then return the top k elements
 %% Cell type:code id: tags:
 ``` python
 query = "Purchased the Lenovo Notebook, and it's been a reliable companion. The design is sleek, and it handles tasks effortlessly. Impressed with the decent battery life, making it suitable for daily use. Overall, a good value for the money.".lower()
 query_result = bert_model.retrieve_top_k_entries_for_query(query=query, k=10)
 print(query_result)
 ```
 %% Output
                                                       text
    7675  great product product is light weight, comfort...
    5077  great value - now these once cost nearly $3000...
    7678  love this, easy to use and makes working from ...
    7676  we like them i purchased two lapinator's plus,...
    7677  much, much better available... i suggest you k...
    7679  it works great! it really works! this is my se...
    7683  great i bought the lapinator and mousitizer af...
    1142  good replacement for old charger this is a goo...
    8969  ladies citizen my wife loves it. got it for he...
    3394  best computer acessory a year and a half ago, ...
 %% Cell type:markdown id: tags:
 ### Sentiment Analysis with BERT
-TODO: Description
+Initialization
+-We create a sentiment-analysis pipeline with our pretrained model bert_sentiment_model.
+To determine the sentiment of a given query and of the most similar reviews to that query we have to function:
+- get_sentiment_of_query for the query
+- get_sentiment_for_each_result for the similar reviews
+In get_sentiment_of_query we return a tuple that contains the sentiment and the sentiment score. Both of those values are returned by the sentiment-analysis pipeline.
+In get_sentiment_for_each_result we return a pd.dataframe that added both values(sentiment and the sentiment score) to a given pd.dataframe. For each of these values a new column is created within the pd.dataframe.
+The higher the sentiment score is, the higher a text tends towards a sentiment.
 %% Cell type:code id: tags:
 ``` python
 query_sentiment = bert_model.get_sentiment_of_query(query=query)
 query_result_with_sentiment = bert_model.get_sentiment_for_each_result(
    results=query_result
 )
 print(
    f"[*] Query ({textwrap.shorten(query, width=30)}): {query_sentiment[0]} [{round(query_sentiment[1] * 100, 4)}%]"
 )
 print(
    "[*] Sentiment distribution of results  : "
    + str(dict(Counter(query_result_with_sentiment.sentiment.tolist())))
 )
 ```
 %% Output
    [*] Query (purchased the lenovo [...]): POSITIVE [99.9848%]
    [*] Sentiment distribution of results  : {'POSITIVE': 8, 'NEGATIVE': 2}
 %% Cell type:markdown id: tags:
 ### Evaluation of the Model
 TODO: Description
 %% Cell type:code id: tags:
 ``` python
 def create_bar_chart(positive_count, negative_count, query_text, score, sentiment):
    terms = ["Positive", "Negative"]
    counts = [positive_count, negative_count]
    bar_width = 0.6
    bar_positions = [0, 1]
    fig, ax = plt.subplots()
    ax.set_title("Sentiment Analysis\n", fontsize=12, weight='bold')
    subheading = f"Query: {query_text} | {sentiment} | Score: {score}%"
    ax.text(0.5, max(counts) + 8, subheading, ha="center", va="center", fontsize=10, weight='light')
    for i, count in enumerate(counts):
            ax.text(bar_positions[i], count + 1, str(count), ha="center", va="bottom", fontsize=8
                    )
    ax.bar(bar_positions, counts, color=["green", "red"], width=bar_width, align="center")
    ax.set_xlabel("Sentiment")
    ax.set_ylabel("Frequency")
    ax.set_xticks(bar_positions)
    ax.set_xticklabels(terms)
    plt.subplots_adjust(bottom=0.2)
    plt.show()
 ```
 %% Cell type:code id: tags:
 ``` python
 # queries = [
 #     "Cruel and Unusual is the first Patricia Cornwell book I have got read and I for one loved it and I can't wait the read more of her books my boyfriend told me about Patricia Cornwell books he said I need to read Hornet's Nest and Southern Cross and those will be my next two book I will be reading and I will be reading them on my new IPad I got for Christmas I can't wait keep up the great writing Patricia and thanks to my boyfriend for telling me about this awesome writer.........RKsbabydoll",
 #     "Actually a good TV. Unfortunately, after a few months the picture is only white. It also cannot be adjusted. The television is therefore junk.",
 #     "I don't really understand the positive reviews here. Yes, the RGB light is good and makes it easier to use in a dark room. The additional buttons are also practical... but seriously...What year is it? The keys on this keyboard remind me of my C64 breadbox: see high and loud keystrokes. The feeling is 1:1 the same.I like flat keyboards myself and still gave this Logitech a chance. But after 30 minutes it was over... I'll definitely never get used to that! Therefore, unfortunately, a reach into the toilet.If there ever is a flat version with a quiet stop, I would be happy to test it again.",
 #     "I recently got the [Laptop Brand/Model] and it's been a game-changer. The sleek design caught my eye, and it performs like a champ—smooth multitasking, vibrant display. Battery life is decent, lasting through my workday. Overall, a solid buy for the price!",
 #     "Purchased the Lenovo Notebook, and it's been a reliable companion. The design is sleek, and it handles tasks effortlessly. Impressed with the decent battery life, making it suitable for daily use. Overall, a good value for the money."
 # ]
 test_dataset = pd.read_csv(
    "data/train_short_200000.csv", names=["sentiment", "title", "text"], sep=","
 ).tail(100)
 test_dataset = test_dataset.dropna()
 test_dataset = test_dataset.drop(["title"], axis=1)
 test_dataset = test_dataset.drop(["sentiment"], axis=1)
 test_dataset.text = test_dataset.text.str.lower()
 number_of_results = 50
 percentages = dict()
 for index, query in test_dataset.iterrows():
    query_sentiment = bert_model.get_sentiment_of_query(query=query.text)
    query_results = bert_model.retrieve_top_k_entries_for_query(
        query=query.text, k=number_of_results
    )
    query_results_with_sentiment = bert_model.get_sentiment_for_each_result(
        results=query_results
    )
    sentiment_distribution = Counter(query_results_with_sentiment.sentiment.tolist())
    percentage = (sentiment_distribution[query_sentiment[0]] / number_of_results) * 100
    percentages[index] = (
        textwrap.shorten(query.text, width=30),
        query_sentiment,
        percentage,
    )
 # print(json.dumps(percentages, indent=4))
 # TODO what to do with all these percentages? How to evaluate/plot them?
 ```
 %% Output
    ---------------------------------------------------------------------------
-    AttributeError                            Traceback (most recent call last)
+    KeyboardInterrupt                         Traceback (most recent call last)
-    /tmp/ipykernel_4859/1683762815.py in ?()
+Cell     In[27], line 25
-         17 number_of_results = 50
+         21 query_sentiment = bert_model.get_sentiment_of_query(query=query.text)
-         18 percentages = dict()
+         22 query_results = bert_model.retrieve_top_k_entries_for_query(
-         19
+         23     query=query.text, k=number_of_results
-         20 for index, query in test_dataset.iterrows():
+         24 )
-    ---> 21     query_sentiment = bert_model.get_sentiment_of_query(query=query.title)
+    ---> 25 query_results_with_sentiment = bert_model.get_sentiment_for_each_result(
-         22     query_results = bert_model.retrieve_top_k_entries_for_query(
+         26     results=query_results
-         23         query=query.text, k=number_of_results
+         27 )
-         24     )
+         29 sentiment_distribution = Counter(query_results_with_sentiment.sentiment.tolist())
-    ~/.local/lib/python3.11/site-packages/pandas/core/generic.py in ?(self, name)
+         30 percentage = (sentiment_distribution[query_sentiment[0]] / number_of_results) * 100
-       6200             and name not in self._accessors
+Cell     In[15], line 33, in BERT.get_sentiment_for_each_result(self, results)
-       6201             and self._info_axis._can_hold_identifiers_and_holds_name(name)
+         31 def get_sentiment_for_each_result(self, results: pd.DataFrame) -> pd.DataFrame:
-       6202         ):
+         32     for index, row in results.iterrows():
-       6203             return self[name]
+    ---> 33         r = self.sentiment_analyzer(row.text)
-    -> 6204         return object.__getattribute__(self, name)
+         34         results.loc[index, "sentiment"] = r[0]["label"]
+         35     return results
-    AttributeError: 'Series' object has no attribute 'title'
+File     ~/.local/lib/python3.11/site-packages/transformers/pipelines/text_classification.py:156, in TextClassificationPipeline.__call__(self, *args, **kwargs)
+        122 def __call__(self, *args, **kwargs):
+        123     """
+        124     Classify the text(s) given as inputs.
+        125
+       (...)
+        154         If `top_k` is used, one such dictionary is returned per label.
+        155     """
+    --> 156     result = super().__call__(*args, **kwargs)
+        157     # TODO try and retrieve it in a nicer way from _sanitize_parameters.
+        158     _legacy = "top_k" not in kwargs
+File     ~/.local/lib/python3.11/site-packages/transformers/pipelines/base.py:1140, in Pipeline.__call__(self, inputs, num_workers, batch_size, *args, **kwargs)
+       1132     return next(
+       1133         iter(
+       1134             self.get_iterator(
+       (...)
+       1137         )
+       1138     )
+       1139 else:
+    -> 1140     return self.run_single(inputs, preprocess_params, forward_params, postprocess_params)
+File     ~/.local/lib/python3.11/site-packages/transformers/pipelines/base.py:1147, in Pipeline.run_single(self, inputs, preprocess_params, forward_params, postprocess_params)
+       1145 def run_single(self, inputs, preprocess_params, forward_params, postprocess_params):
+       1146     model_inputs = self.preprocess(inputs, **preprocess_params)
+    -> 1147     model_outputs = self.forward(model_inputs, **forward_params)
+       1148     outputs = self.postprocess(model_outputs, **postprocess_params)
+       1149     return outputs
+File     ~/.local/lib/python3.11/site-packages/transformers/pipelines/base.py:1046, in Pipeline.forward(self, model_inputs, **forward_params)
+       1044     with inference_context():
+       1045         model_inputs = self._ensure_tensor_on_device(model_inputs, device=self.device)
+    -> 1046         model_outputs = self._forward(model_inputs, **forward_params)
+       1047         model_outputs = self._ensure_tensor_on_device(model_outputs, device=torch.device("cpu"))
+       1048 else:
+File     ~/.local/lib/python3.11/site-packages/transformers/pipelines/text_classification.py:187, in TextClassificationPipeline._forward(self, model_inputs)
+        185 if "use_cache" in inspect.signature(model_forward).parameters.keys():
+        186     model_inputs["use_cache"] = False
+    --> 187 return self.model(**model_inputs)
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
+       1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
+       1517 else:
+    -> 1518     return self._call_impl(*args, **kwargs)
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
+       1522 # If we don't have any hooks, we want to skip the rest of the logic in
+       1523 # this function, and just call forward.
+       1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
+       1525         or _global_backward_pre_hooks or _global_backward_hooks
+       1526         or _global_forward_hooks or _global_forward_pre_hooks):
+    -> 1527     return forward_call(*args, **kwargs)
+       1529 try:
+       1530     result = None
+File     ~/.local/lib/python3.11/site-packages/transformers/models/distilbert/modeling_distilbert.py:1000, in DistilBertForSequenceClassification.forward(self, input_ids, attention_mask, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
+        992 r"""
+        993 labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+        994     Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+        995     config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+        996     `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        997 """
+        998 return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+    -> 1000 distilbert_output = self.distilbert(
+       1001     input_ids=input_ids,
+       1002     attention_mask=attention_mask,
+       1003     head_mask=head_mask,
+       1004     inputs_embeds=inputs_embeds,
+       1005     output_attentions=output_attentions,
+       1006     output_hidden_states=output_hidden_states,
+       1007     return_dict=return_dict,
+       1008 )
+       1009 hidden_state = distilbert_output[0]  # (bs, seq_len, dim)
+       1010 pooled_output = hidden_state[:, 0]  # (bs, dim)
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
+       1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
+       1517 else:
+    -> 1518     return self._call_impl(*args, **kwargs)
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
+       1522 # If we don't have any hooks, we want to skip the rest of the logic in
+       1523 # this function, and just call forward.
+       1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
+       1525         or _global_backward_pre_hooks or _global_backward_hooks
+       1526         or _global_forward_hooks or _global_forward_pre_hooks):
+    -> 1527     return forward_call(*args, **kwargs)
+       1529 try:
+       1530     result = None
+File     ~/.local/lib/python3.11/site-packages/transformers/models/distilbert/modeling_distilbert.py:820, in DistilBertModel.forward(self, input_ids, attention_mask, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)
+        817     if attention_mask is None:
+        818         attention_mask = torch.ones(input_shape, device=device)  # (bs, seq_length)
+    --> 820 return self.transformer(
+        821     x=embeddings,
+        822     attn_mask=attention_mask,
+        823     head_mask=head_mask,
+        824     output_attentions=output_attentions,
+        825     output_hidden_states=output_hidden_states,
+        826     return_dict=return_dict,
+        827 )
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
+       1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
+       1517 else:
+    -> 1518     return self._call_impl(*args, **kwargs)
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
+       1522 # If we don't have any hooks, we want to skip the rest of the logic in
+       1523 # this function, and just call forward.
+       1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
+       1525         or _global_backward_pre_hooks or _global_backward_hooks
+       1526         or _global_forward_hooks or _global_forward_pre_hooks):
+    -> 1527     return forward_call(*args, **kwargs)
+       1529 try:
+       1530     result = None
+File     ~/.local/lib/python3.11/site-packages/transformers/models/distilbert/modeling_distilbert.py:585, in Transformer.forward(self, x, attn_mask, head_mask, output_attentions, output_hidden_states, return_dict)
+        577     layer_outputs = self._gradient_checkpointing_func(
+        578         layer_module.__call__,
+        579         hidden_state,
+       (...)
+        582         output_attentions,
+        583     )
+        584 else:
+    --> 585     layer_outputs = layer_module(
+        586         hidden_state,
+        587         attn_mask,
+        588         head_mask[i],
+        589         output_attentions,
+        590     )
+        592 hidden_state = layer_outputs[-1]
+        594 if output_attentions:
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
+       1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
+       1517 else:
+    -> 1518     return self._call_impl(*args, **kwargs)
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
+       1522 # If we don't have any hooks, we want to skip the rest of the logic in
+       1523 # this function, and just call forward.
+       1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
+       1525         or _global_backward_pre_hooks or _global_backward_hooks
+       1526         or _global_forward_hooks or _global_forward_pre_hooks):
+    -> 1527     return forward_call(*args, **kwargs)
+       1529 try:
+       1530     result = None
+File     ~/.local/lib/python3.11/site-packages/transformers/models/distilbert/modeling_distilbert.py:530, in TransformerBlock.forward(self, x, attn_mask, head_mask, output_attentions)
+        528 # Feed Forward Network
+        529 ffn_output = self.ffn(sa_output)  # (bs, seq_length, dim)
+    --> 530 ffn_output: torch.Tensor = self.output_layer_norm(ffn_output + sa_output)  # (bs, seq_length, dim)
+        532 output = (ffn_output,)
+        533 if output_attentions:
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
+       1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
+       1517 else:
+    -> 1518     return self._call_impl(*args, **kwargs)
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
+       1522 # If we don't have any hooks, we want to skip the rest of the logic in
+       1523 # this function, and just call forward.
+       1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
+       1525         or _global_backward_pre_hooks or _global_backward_hooks
+       1526         or _global_forward_hooks or _global_forward_pre_hooks):
+    -> 1527     return forward_call(*args, **kwargs)
+       1529 try:
+       1530     result = None
+File     ~/.local/lib/python3.11/site-packages/torch/nn/modules/normalization.py:196, in LayerNorm.forward(self, input)
+        195 def forward(self, input: Tensor) -> Tensor:
+    --> 196     return F.layer_norm(
+        197         input, self.normalized_shape, self.weight, self.bias, self.eps)
+File     ~/.local/lib/python3.11/site-packages/torch/nn/functional.py:2543, in layer_norm(input, normalized_shape, weight, bias, eps)
+       2539 if has_torch_function_variadic(input, weight, bias):
+       2540     return handle_torch_function(
+       2541         layer_norm, (input, weight, bias), input, normalized_shape, weight=weight, bias=bias, eps=eps
+       2542     )
+    -> 2543 return torch.layer_norm(input, normalized_shape, weight, bias, eps, torch.backends.cudnn.enabled)
+    KeyboardInterrupt:
 %% Cell type:code id: tags:
 ``` python
 highest_positive_query = ("", "", 0)
 highest_negative_query = ("", "", 0)
 smallest_positive_query = ("", "", 100)
 smallest_negative_query = ("", "", 100)
 for query in percentages:
    if percentages[query][1][0] == "POSITIVE":
        if percentages[query][2] > highest_positive_query[2]:
            highest_positive_query = percentages[query]
        elif percentages[query][2] < smallest_positive_query[2]:
            smallest_positive_query = percentages[query]
    else:
        if percentages[query][2] > highest_negative_query[2]:
            highest_negative_query = percentages[query]
        elif percentages[query][2] < smallest_negative_query[2]:
            smallest_negative_query = percentages[query]
 print(f"Highest positive query: {highest_positive_query}")
 print(f"Highest negative query: {highest_negative_query}")
 print(f"Lowest positive query: {smallest_positive_query}")
 print(f"Lowest negative query: {smallest_negative_query}")
 create_bar_chart(
    positive_count=highest_positive_query[2],
    negative_count=100 - highest_positive_query[2],
    query_text=highest_positive_query[0],
    score=round(highest_positive_query[1][1] * 100, 4),
    sentiment=highest_positive_query[1][0],
 )
 create_bar_chart(
    positive_count=100 - highest_negative_query[2],
    negative_count=highest_negative_query[2],
    query_text=highest_negative_query[0],
    score=round(highest_negative_query[1][1] * 100, 4),
    sentiment=highest_negative_query[1][0],
 )
 create_bar_chart(
    positive_count=smallest_positive_query[2],
    negative_count=100 - smallest_positive_query[2],
    query_text=smallest_positive_query[0],
    score=round(smallest_positive_query[1][1] * 100, 4),
    sentiment=smallest_positive_query[1][0],
 )
 create_bar_chart(
    positive_count=100 - smallest_negative_query[2],
    negative_count=smallest_negative_query[2],
    query_text=smallest_negative_query[0],
    score=round(smallest_negative_query[1][1] * 100, 4),
    sentiment=smallest_negative_query[1][0],
 )
 ```
 %% Output
    Highest positive query: ('very nice movie , it [...]', ('POSITIVE', 0.9998714923858643), 90.0)
    Highest negative query: ('i am returning my copy. [...]', ('NEGATIVE', 0.9997848868370056), 94.0)
    Lowest positive query: ('this was so terrible, i [...]', ('POSITIVE', 0.997624933719635), 6.0)
    Lowest negative query: ("i'm preparing the capes [...]", ('NEGATIVE', 0.5031839609146118), 22.0)
 %% Cell type:code id: tags:
 ``` python
 total = 0
 for index, percentage in percentages.items():
    total += percentage[2]
 print(total / len(percentages))
 ```
 %% Output
    61.28