update

hanlv15 · Jul 14, 2024 · 1665378 · 1665378
1 parent 7dd241a
commit 1665378
Show file tree

Hide file tree

Showing 12 changed files with 560 additions and 47 deletions.
diff --git a/examples/pytorch/llm/my_inferencing/infer_tuned.py b/examples/pytorch/llm/my_inferencing/infer_tuned.py
@@ -72,15 +72,17 @@ def get_model_template():
     model_type, template_type = sft_args["model_type"], sft_args["template_type"]
     model, tokenizer = get_model_tokenizer(
         model_type, model_kwargs={'device_map': 'auto'},
-        model_dir=sft_args["model_cache_dir"],
+        # model_dir=sft_args["model_cache_dir"],
         use_flash_attn=sft_args["use_flash_attn"]
     )
     model = Swift.from_pretrained(model, ckpt_dir, inference_mode=True)
     if sft_args["sft_type"] == 'adalora':
         model = model.to(model.dtype)
     model.generation_config.max_new_tokens = 512
+    # model.generation_config.temperature = None
     model.generation_config.do_sample = False
 
+
     template = get_template(template_type, tokenizer)
 
     return model, template

diff --git a/examples/pytorch/llm/my_inferencing/label_check/check_dataset.ipynb b/examples/pytorch/llm/my_inferencing/label_check/check_dataset.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -28,7 +28,7 @@
     "        json.dump(data, f, indent=4)\n",
     "\n",
     "def load_train_bak():\n",
-    "    with open(\"covmis_data/train.json.bak\", \"r\") as f:\n",
+    "    with open(\"/home/hanlv/workspace/data/machine_learning/dataset/research/misinformation_dataset/COVMIS-main/data/train.json\", \"r\") as f:\n",
     "        return json.load(f)\n",
     "    \n",
     "def load_train():\n",
@@ -14976,28 +14976,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(177, 2439)"
+       "(0.924559245592456, 0.8867710877419616, 0.9524633448215433, 0.849041520589057)"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 2,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
+    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
     "DEFAULT_SEED = 42\n",
     "import numpy as np\n",
     "import random\n",
     "\n",
     "def set_seed(seed=DEFAULT_SEED):\n",
     "    np.random.seed(seed)\n",
     "    random.seed(seed)\n",
+    "set_seed()\n",
     "\n",
     "data_train = load_train()\n",
     "data_train_bak = load_train_bak()\n",
@@ -15006,7 +15010,6 @@
     "for item in data_train_bak:\n",
     "    label_old[item[\"id\"]] = item[\"label\"]\n",
     "\n",
-    "from sklearn.model_selection import train_test_split\n",
     "\n",
     "data_train_new = []\n",
     "for item in data_train:\n",
@@ -15016,10 +15019,32 @@
     "train_list, test_list = train_test_split(data_train_new, test_size=0.2, shuffle=True)\n",
     "\n",
     "cnt = 0\n",
+    "labels, preds = [], [] # preds为covmis原始标签\n",
     "for item in test_list:\n",
-    "    if label_old[item[\"id\"]] != item[\"label\"]:\n",
+    "    if int(label_old[item[\"id\"]]) == 2:\n",
+    "        preds.append(1)\n",
+    "    elif int(label_old[item[\"id\"]]) == 0:\n",
+    "        preds.append(0)\n",
+    "    else:\n",
+    "        raise Exception()\n",
+    "\n",
+    "    if item[\"label\"] == 2:\n",
+    "        labels.append(1)\n",
+    "    elif item[\"label\"] == 0:\n",
+    "        labels.append(0)\n",
+    "    else:\n",
+    "        \n",
+    "        raise Exception()\n",
+    "    \n",
+    "    if int(label_old[item[\"id\"]]) != item[\"label\"]:\n",
     "        cnt += 1\n",
-    "cnt, len(test_list)"
+    "\n",
+    "acc = accuracy_score(labels, preds)\n",
+    "f1 = f1_score(labels, preds, average='macro')\n",
+    "precision = precision_score(labels, preds, average='macro')\n",
+    "recall = recall_score(labels, preds, average='macro')\n",
+    "\n",
+    "acc, f1, precision, recall\n"
    ]
   },
   {

diff --git a/examples/pytorch/llm/my_inferencing/run_infer.py b/examples/pytorch/llm/my_inferencing/run_infer.py
@@ -1,16 +1,12 @@
 ckpt_list = [
-    # "/home/hanlv/workspace/code/research/infodemic/LLM/swift/examples/pytorch/llm/output/openchat_3.5/with_llama3_info/brave/data1-split=8:2-ratio=1.0/rslora-r=8/lr=1e-5-20240711-10:28:24/checkpoint-609",
-    # "/home/hanlv/workspace/code/research/infodemic/LLM/swift/examples/pytorch/llm/output/openchat_3.5/with_llama3_info/brave/data1-split=8:2-ratio=1.0/rslora-r=8/lr=3e-5-20240711-10:37:24/checkpoint-609",
-    "/home/hanlv/workspace/code/research/infodemic/LLM/swift/examples/pytorch/llm/output/openchat_3.5/with_llama3_info/brave/data1-split=8:2-ratio=1.0/rslora-r=8/lr=5e-5-20240711-10:31:53/checkpoint-609",
-
 
 ]
 
-path = "/home/hanlv/workspace/code/research/infodemic/LLM/swift/examples/pytorch/llm/output/openchat_3.5/with_llama3_info/brave/data1-split=8:2-ratio=1.0/lora-r=8"
+path = "/home/hanlv/workspace/code/research/infodemic/LLM/swift/examples/pytorch/llm/output/Mistral-7B-Instruct-v0.3/with_llama3_info/brave/data1-split=8:2-ratio=1.0/vera-r=256"
 # path = "/home/hanlv/workspace/code/research/infodemic/LLM/swift/examples/pytorch/llm/output/Mistral-7B-Instruct-v0.3/with_llama3_info/brave/data1-split=8:2-ratio=1.0/lora_plus-r=8"
 # path = "/home/hanlv/workspace/code/research/infodemic/LLM/swift/examples/pytorch/llm/output/Mistral-7B-Instruct-v0.3/with_llama3_info/brave/data1-split=8:2-ratio=1.0/dora-r=8"
 
-path = ""
+# path = ""
 
 import subprocess
 import os

diff --git a/...lama3_info/brave/data1-split=8:2-ratio=1.0/dora-r=8/Mistral-7B-Instruct-v0.3(_llama).json b/...lama3_info/brave/data1-split=8:2-ratio=1.0/dora-r=8/Mistral-7B-Instruct-v0.3(_llama).json
@@ -21,6 +21,17 @@
         "Precision": 0.9862165023121066,
         "Recall": 0.9719179501969232
     },
+    {
+        "model": "Mistral-7B-Instruct-v0.3",
+        "train_test_split": "8:2",
+        "train_ratio": "1.0",
+        "train_loss": 0.07149822,
+        "lr": "6.5e-5",
+        "ACC": 0.984829848298483,
+        "F1": 0.979355026154235,
+        "Precision": 0.9870772122737896,
+        "Recall": 0.9721905783321467
+    },
     {
         "model": "Mistral-7B-Instruct-v0.3",
         "train_test_split": "8:2",
@@ -120,6 +131,17 @@
         "Precision": 0.9861662515039682,
         "Recall": 0.9763313716124264
     },
+    {
+        "model": "Mistral-7B-Instruct-v0.3",
+        "train_test_split": "8:2",
+        "train_ratio": "1.0",
+        "train_loss": 0.07067327,
+        "lr": "1.05e-4",
+        "ACC": 0.986059860598606,
+        "F1": 0.9810831896197749,
+        "Precision": 0.9873110989530964,
+        "Recall": 0.9752237353208901
+    },
     {
         "model": "Mistral-7B-Instruct-v0.3",
         "train_test_split": "8:2",
@@ -131,6 +153,17 @@
         "Precision": 0.9853237080893946,
         "Recall": 0.9760587434772028
     },
+    {
+        "model": "Mistral-7B-Instruct-v0.3",
+        "train_test_split": "8:2",
+        "train_ratio": "1.0",
+        "train_loss": 0.08678514,
+        "lr": "1.15e-4",
+        "ACC": 0.984419844198442,
+        "F1": 0.978784857274705,
+        "Precision": 0.9868058050548505,
+        "Recall": 0.971364132051155
+    },
     {
         "model": "Mistral-7B-Instruct-v0.3",
         "train_test_split": "8:2",

diff --git a/..._info/brave/data1-split=8:2-ratio=1.0/lora_plus-r=8/Mistral-7B-Instruct-v0.3(_llama).json b/..._info/brave/data1-split=8:2-ratio=1.0/lora_plus-r=8/Mistral-7B-Instruct-v0.3(_llama).json
@@ -1,4 +1,15 @@
 [
+    {
+        "model": "Mistral-7B-Instruct-v0.3",
+        "train_test_split": "8:2",
+        "train_ratio": "1.0",
+        "train_loss": 0.07841515,
+        "lr": "4.5e-6",
+        "ACC": 0.9868798687986879,
+        "F1": 0.9821755803989849,
+        "Precision": 0.9890226753546556,
+        "Recall": 0.9757689915913372
+    },
     {
         "model": "Mistral-7B-Instruct-v0.3",
         "train_test_split": "8:2",
@@ -10,6 +21,17 @@
         "Precision": 0.9887110918288224,
         "Recall": 0.9771492560180971
     },
+    {
+        "model": "Mistral-7B-Instruct-v0.3",
+        "train_test_split": "8:2",
+        "train_ratio": "1.0",
+        "train_loss": 0.07524356,
+        "lr": "5.5e-6",
+        "ACC": 0.985239852398524,
+        "F1": 0.9799475279488581,
+        "Precision": 0.986763291826386,
+        "Recall": 0.9735708427589067
+    },
     {
         "model": "Mistral-7B-Instruct-v0.3",
         "train_test_split": "8:2",
@@ -21,6 +43,17 @@
         "Precision": 0.9845039082981164,
         "Recall": 0.971372693926476
     },
+    {
+        "model": "Mistral-7B-Instruct-v0.3",
+        "train_test_split": "8:2",
+        "train_ratio": "1.0",
+        "train_loss": 0.07073201,
+        "lr": "6.5e-6",
+        "ACC": 0.985239852398524,
+        "F1": 0.9799475279488581,
+        "Precision": 0.986763291826386,
+        "Recall": 0.9735708427589067
+    },
     {
         "model": "Mistral-7B-Instruct-v0.3",
         "train_test_split": "8:2",
@@ -32,6 +65,17 @@
         "Precision": 0.9876208009497134,
         "Recall": 0.9738434708941301
     },
+    {
+        "model": "Mistral-7B-Instruct-v0.3",
+        "train_test_split": "8:2",
+        "train_ratio": "1.0",
+        "train_loss": 0.07158582,
+        "lr": "7.5e-6",
+        "ACC": 0.986059860598606,
+        "F1": 0.9810615541739214,
+        "Precision": 0.9878929835905208,
+        "Recall": 0.9746699171751219
+    },
     {
         "model": "Mistral-7B-Instruct-v0.3",
         "train_test_split": "8:2",
@@ -43,6 +87,17 @@
         "Precision": 0.9856707254643169,
         "Recall": 0.9702650576349396
     },
+    {
+        "model": "Mistral-7B-Instruct-v0.3",
+        "train_test_split": "8:2",
+        "train_ratio": "1.0",
+        "train_loss": 0.06904706,
+        "lr": "8.5e-6",
+        "ACC": 0.985649856498565,
+        "F1": 0.9805379163521399,
+        "Precision": 0.9864596795037477,
+        "Recall": 0.9749511071856665
+    },
     {
         "model": "Mistral-7B-Instruct-v0.3",
         "train_test_split": "8:2",
@@ -54,6 +109,17 @@
         "Precision": 0.988134728481993,
         "Recall": 0.9777030741638653
     },
+    {
+        "model": "Mistral-7B-Instruct-v0.3",
+        "train_test_split": "8:2",
+        "train_ratio": "1.0",
+        "train_loss": 0.06740927,
+        "lr": "9.5e-6",
+        "ACC": 0.984419844198442,
+        "F1": 0.9788335017237946,
+        "Precision": 0.9856336000622512,
+        "Recall": 0.9724717683426913
+    },
     {
         "model": "Mistral-7B-Instruct-v0.3",
         "train_test_split": "8:2",