{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 排卵试纸机器学习算法验证" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 1. **import moudle**" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd \n", "import seaborn as sns\n", "from IPython.display import display\n", "import matplotlib.pyplot as plt\n", "from mpl_toolkits.mplot3d import Axes3D\n", "import sklearn\n", "%matplotlib inline\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 2. **load data**" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "load data successful !!!!!\n" ] } ], "source": [ "try :\n", " data1 = pd.read_csv(\"ov2020-5-14/ovulation-4e-5-1.csv\")\n", " data2 = pd.read_csv(\"ov2020-5-14/ovulation-4e-10-1.csv\")\n", " data3 = pd.read_csv(\"ov2020-5-14/ovulation-4e-10-2.csv\")\n", " data4 = pd.read_csv(\"ov2020-5-14/ovulation-4e-15-1.csv\")\n", " data5 = pd.read_csv(\"ov2020-5-14/ovulation-4e-15-2.csv\")\n", " data6 = pd.read_csv(\"ov2020-5-14/ovulation-4e-25-1.csv\")\n", " data7 = pd.read_csv(\"ov2020-5-14/ovulation-4e-25-2.csv\")\n", " data8 = pd.read_csv(\"ov2020-5-14/ovulation-4e-40-1.csv\")\n", " data9 = pd.read_csv(\"ov2020-5-14/ovulation-4e-50-1.csv\")\n", " data10 = pd.read_csv(\"ov2020-5-14/ovulation-4e-50-2.csv\")\n", " data11 = pd.read_csv(\"ov2020-5-14/ovulation-4e-50-3.csv\")\n", " data12 = pd.read_csv(\"ov2020-5-14/ovulation-4e-75-1.csv\")\n", " data13 = pd.read_csv(\"ov2020-5-14/ovulation-4e-75-2.csv\")\n", " data14 = pd.read_csv(\"ov2020-5-14/ovulation-4e-75-3.csv\")\n", " data15 = pd.read_csv(\"ov2020-5-14/ovulation-4e-75-4.csv\")\n", " data16 = pd.read_csv(\"ov2020-5-14/ovulation-4e-75-5.csv\")\n", " data17 = pd.read_csv(\"ov2020-5-14/ovulation-5i-5-1.csv\")\n", " data18 = pd.read_csv(\"ov2020-5-14/ovulation-5i-10-1.csv\")\n", " data19 = pd.read_csv(\"ov2020-5-14/ovulation-5i-10-2.csv\")\n", " data20 = pd.read_csv(\"ov2020-5-14/ovulation-5i-15-1.csv\")\n", " data21 = pd.read_csv(\"ov2020-5-14/ovulation-5i-15-2.csv\")\n", " data22 = pd.read_csv(\"ov2020-5-14/ovulation-5i-25-1.csv\")\n", " data23 = pd.read_csv(\"ov2020-5-14/ovulation-5i-25-2.csv\")\n", " data24 = pd.read_csv(\"ov2020-5-14/ovulation-5i-40-1.csv\")\n", " data25 = pd.read_csv(\"ov2020-5-14/ovulation-5i-50-1.csv\")\n", " data26 = pd.read_csv(\"ov2020-5-14/ovulation-5i-50-2.csv\")\n", " data27 = pd.read_csv(\"ov2020-5-14/ovulation-5i-50-3.csv\")\n", " data28 = pd.read_csv(\"ov2020-5-14/ovulation-5i-75-1.csv\")\n", " data29 = pd.read_csv(\"ov2020-5-14/ovulation-5i-75-2.csv\")\n", " data30 = pd.read_csv(\"ov2020-5-14/ovulation-5i-75-3.csv\")\n", " \n", " print (\"load data successful !!!!!\")\n", "except :\n", " print (\"load data error !!!!!!!!!!\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "C:\\Users\\lenovo\\AppData\\Local\\Temp\\ipykernel_2204\\3906487309.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n" ] } ], "source": [ "data = data1.append(data2).append(data3).append(data4).append(data5).append(data6).append(data7).append(data8).append(data9).append(data10).append(data11).append(data12).append(data13).append(data14).append(data15).append(data16).append(data17).append(data18).append(data19).append(data20).append(data21).append(data22).append(data23).append(data24).append(data25).append(data26).append(data27).append(data28).append(data29).append(data30)\n", "#data10_all['index'].replace(2,1,inplace=True)\n", "data.describe()\n", "data.to_excel('data_all_20230720.xlsx')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "try :\n", "# data_iphone6p_75_10 = pd.read_csv(\"20170912.pm.csv\")\n", "# data_iphone6p_1234 = pd.read_csv(\"20170920.pm.csv\")\n", "# data_iphone6p_5 = pd.read_csv(\"20170922.pm.csv\")\n", "# data_iphone6p_0 = pd.read_csv(\"20170925.am.csv\")\n", "# data_iphone6p_0_0 = pd.read_csv(\"20170925.pm.csv\")\n", "# data_iphone6p_246 = pd.read_csv(\"20171011.pm.csv\")\n", " \n", "# data1 = pd.read_csv(\"ovdata_reindex.csv\")\n", "# data2 = pd.read_csv(\"ovdataMore_reindex.csv\")\n", "# data3 = pd.read_csv(\"ov_data_2020_reindex.csv\")\n", " ovdata = pd.read_csv(\"ovdata.csv\")\n", " ovdataMore = pd.read_csv(\"ovdataMore.csv\")\n", " ov_data_2020 = pd.read_csv(\"ov_data_2020.csv\")\n", " data10more = pd.read_csv(\"data10more.csv\")\n", "\n", "# data4 = pd.read_csv(\"10_25_renew.csv\")\n", "\n", "# data_all = pd.read_csv(\"data_all_2019_2020_reindex.csv\")\n", "# data_all = pd.read_csv(\"ov_data_2020_reindex.csv\")\n", " \n", "# data1 = pd.read_csv(\"ovdata.csv\")\n", "# data2 = pd.read_csv(\"ovdataMore.csv\")\n", "# data3 = pd.read_csv(\"ov_data_2020.csv\")\n", "# data_test1 = pd.read_csv(\"./newData/test.csv\")\n", "# data_test2 = pd.read_csv(\"./newData/nubia_test.csv\")\n", " \n", " print (\"load data successful !!!!!\")\n", "except :\n", " print (\"load data error !!!!!!!!!!\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "data2019_2020_old = ovdata.append(ovdataMore).append(ov_data_2020).append(data10more)\n", "data2019_2020_old['index'].replace(4,7,inplace=True)\n", "data2019_2020_old['index'].replace(3,6,inplace=True)\n", "data2019_2020_old['index'].replace(2,4,inplace=True)\n", "data2019_2020_old['index'].replace(1,2,inplace=True)\n", "\n", "data2019_2020_old.describe()\n", "data2019_2020_old.to_excel('data2019_2020_old.xlsx')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "load data successful !!!!!\n" ] } ], "source": [ "try :\n", "# data_iphone6p_75_10 = pd.read_csv(\"20170912.pm.csv\")\n", "# data_iphone6p_1234 = pd.read_csv(\"20170920.pm.csv\")\n", "# data_iphone6p_5 = pd.read_csv(\"20170922.pm.csv\")\n", "# data_iphone6p_0 = pd.read_csv(\"20170925.am.csv\")\n", "# data_iphone6p_0_0 = pd.read_csv(\"20170925.pm.csv\")\n", "# data_iphone6p_246 = pd.read_csv(\"20171011.pm.csv\")\n", " \n", "# data1 = pd.read_csv(\"ovdata_reindex.csv\")\n", "# data2 = pd.read_csv(\"ovdataMore_reindex.csv\")\n", "# data3 = pd.read_csv(\"ov_data_2020_reindex.csv\")\n", " d1 = pd.read_excel(\"data_all_2020514.xlsx\")\n", " d2 = pd.read_excel(\"data2019_2020_old.xlsx\")\n", "# data4 = pd.read_csv(\"10_25_renew.csv\")\n", "\n", "# data_all = pd.read_csv(\"data_all_2019_2020_reindex.csv\")\n", "# data_all = pd.read_csv(\"ov_data_2020_reindex.csv\")\n", " \n", "# data1 = pd.read_csv(\"ovdata.csv\")\n", "# data2 = pd.read_csv(\"ovdataMore.csv\")\n", "# data3 = pd.read_csv(\"ov_data_2020.csv\")\n", "# data_test1 = pd.read_csv(\"./newData/test.csv\")\n", "# data_test2 = pd.read_csv(\"./newData/nubia_test.csv\")\n", " \n", " print (\"load data successful !!!!!\")\n", "except :\n", " print (\"load data error !!!!!!!!!!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 3. **分析数据**" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Unnamed: 0 left_block_R left_block_G left_block_B left_block_H \\\n", "count 82180.000000 82180.000000 82180.000000 82180.000000 82180.000000 \n", "mean 8668.489827 185.033086 156.928292 166.615296 198.278206 \n", "std 9514.701850 24.755951 33.981587 31.208994 66.447509 \n", "min 0.000000 43.000000 30.000000 33.000000 0.000000 \n", "25% 1294.000000 165.000000 130.000000 142.000000 193.000000 \n", "50% 3917.500000 187.000000 158.000000 169.000000 227.000000 \n", "75% 14267.250000 204.000000 188.000000 193.000000 239.000000 \n", "max 36231.000000 255.000000 255.000000 255.000000 250.000000 \n", "\n", " left_block_S left_block_V left_block_l left_block_a left_block_b \\\n", "count 82180.000000 82180.000000 82180.000000 82180.000000 82180.000000 \n", "mean 43.749002 185.842529 171.646495 139.822962 126.542602 \n", "std 23.980098 24.836509 29.823830 6.400393 3.844306 \n", "min 0.000000 43.000000 34.000000 121.000000 111.000000 \n", "25% 24.000000 167.000000 150.000000 135.000000 125.000000 \n", "50% 41.000000 187.000000 173.000000 140.000000 127.000000 \n", "75% 63.000000 205.000000 198.000000 145.000000 129.000000 \n", "max 148.000000 255.000000 255.000000 154.000000 144.000000 \n", "\n", " ... right_grayHist right_grayMax right_grayMin white_grayValue \\\n", "count ... 82180.000000 82180.000000 82180.000000 82180.000000 \n", "mean ... 162.055135 197.940302 133.605890 205.953212 \n", "std ... 25.113326 15.384508 23.201433 16.090321 \n", "min ... 28.000000 91.000000 24.000000 102.000000 \n", "25% ... 144.000000 190.000000 115.000000 197.000000 \n", "50% ... 162.000000 199.000000 135.000000 205.000000 \n", "75% ... 180.000000 208.000000 152.000000 218.000000 \n", "max ... 251.000000 255.000000 242.000000 255.000000 \n", "\n", " white_grayStddevValue white_grayHist white_grayMax white_grayMin \\\n", "count 82180.000000 82180.000000 82180.000000 82180.000000 \n", "mean 0.185763 206.318885 207.747846 205.103663 \n", "std 0.467831 16.578652 15.995001 16.190066 \n", "min 0.000000 0.000000 103.000000 101.000000 \n", "25% 0.000000 197.000000 199.000000 195.000000 \n", "50% 0.000000 205.000000 206.000000 204.000000 \n", "75% 0.000000 218.000000 219.000000 217.000000 \n", "max 17.000000 254.000000 255.000000 255.000000 \n", "\n", " whiteBalance index \n", "count 82180.0 82180.000000 \n", "mean 0.0 4.072791 \n", "std 0.0 2.285574 \n", "min 0.0 0.000000 \n", "25% 0.0 2.000000 \n", "50% 0.0 4.000000 \n", "75% 0.0 6.000000 \n", "max 0.0 7.000000 \n", "\n", "[8 rows x 153 columns]\n" ] } ], "source": [ "# data4 = data_iphone6p_246[data_iphone6p_246[\"whiteBalance\"] == 0]\n", "# data2= data_iphone6p_1234[data_iphone6p_1234[\"whiteBalance\"] == 0 ]\n", "# data1 = data_iphone6p_75_10[data_iphone6p_75_10[\"whiteBalance\"] == 0 ]\n", "# data3 = data_iphone6p_5[data_iphone6p_5[\"whiteBalance\"] == 0]\n", "# data0 = data_iphone6p_0[data_iphone6p_0[\"whiteBalance\"] == 0]\n", "# data0_0 = data_iphone6p_0_0[data_iphone6p_0_0[\"whiteBalance\"] == 0]\n", "\n", "\n", "#data_all = data2.append(data1[data1[\"index\"] == 5 ]).append(data3).append(data1[data1[\"index\"] == 7 ]).append(data1[data1[\"index\"] == 8 ]).append(data0).append(data0_0).append(data4)\n", "#data1['index'].replace(4,6,inplace=True)\n", "#data1['index'].replace(3,5,inplace=True)\n", "#data1['index'].replace(2,4,inplace=True)\n", "#data1['index'].replace(1,2,inplace=True)\n", "\n", "#data2['index'].replace(4,6,inplace=True)\n", "#data2['index'].replace(3,5,inplace=True)\n", "#data2['index'].replace(2,4,inplace=True)\n", "#data2['index'].replace(1,2,inplace=True)\n", "\n", "#data3['index'].replace(4,6,inplace=True)\n", "#data3['index'].replace(3,5,inplace=True)\n", "#data3['index'].replace(2,4,inplace=True)\n", "#data3['index'].replace(1,2,inplace=True)\n", "\n", "#data4['index'].replace(2,1,inplace=True)\n", "#data4['index'].replace(4,2,inplace=True)\n", "\n", "#data1_0 = data1[data1[\"whiteBalance\"] == 0]\n", "#data2_0 = data2[data2[\"whiteBalance\"] == 0]\n", "#data3_0 = data3[data3[\"whiteBalance\"] == 0]\n", "\n", "#data_test_0 = data_test\n", "\n", "#data_all =data1_0.append(data2_0);\n", "#data_all =data1.append(data2).append(data3);\n", "\n", "#data_all.to_csv('data_all_2019_2020_reindex.csv')\n", "#data1.to_csv('ovdata_modifed.csv')\n", "#data2.to_csv('ovdataMore_modifed.csv')\n", "#data3.to_csv('ov_data_2020_modifed.csv')\n", "#data4.to_csv('10_25_renew.csv')\n", "\n", "data =d1.append(d2)#.append(data3).append(data4)\n", "data_all = data[data[\"whiteBalance\"] == 0]\n", "print(data_all.describe())\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "whiteBlock_R_one =data_all[data_all[\"index\"] == 0 ][\"right_block_l_min\"]\n", "whiteBlock_G_one = data_all[data_all[\"index\"] == 0 ][\"right_block_a_min\"]\n", "whiteBlock_B_one = data_all[data_all[\"index\"] == 0 ][\"right_block_b_min\"]\n", "\n", "whiteBlock_R_two = data_all[data_all[\"index\"] == 1 ][\"right_block_l_min\"]\n", "whiteBlock_G_two = data_all[data_all[\"index\"] == 1 ][\"right_block_a_min\"]\n", "whiteBlock_B_two = data_all[data_all[\"index\"] == 1 ][\"right_block_b_min\"]\n", "\n", "whiteBlock_R_three = data_all[data_all[\"index\"] == 2 ][\"right_block_l_min\"]\n", "whiteBlock_G_three = data_all[data_all[\"index\"] == 2 ][\"right_block_a_min\"]\n", "whiteBlock_B_three = data_all[data_all[\"index\"] == 2 ][\"right_block_b_min\"]\n", "\n", "whiteBlock_R_four = data_all[data_all[\"index\"] == 4 ][\"right_block_l_min\"]\n", "whiteBlock_G_four = data_all[data_all[\"index\"] == 4 ][\"right_block_a_min\"]\n", "whiteBlock_B_four = data_all[data_all[\"index\"] == 4 ][\"right_block_b_min\"]\n", "\n", "\n", "whiteBlock_R_five = data_all[data_all[\"index\"] == 6 ][\"right_block_l_min\"]\n", "whiteBlock_G_five = data_all[data_all[\"index\"] == 6 ][\"right_block_a_min\"]\n", "whiteBlock_B_five = data_all[data_all[\"index\"] == 6 ][\"right_block_b_min\"]\n", "\n", "whiteBlock_R_six = data_all[data_all[\"index\"] == 7 ][\"right_block_l_min\"]\n", "whiteBlock_G_six = data_all[data_all[\"index\"] == 7 ][\"right_block_a_min\"]\n", "whiteBlock_B_six = data_all[data_all[\"index\"] == 7 ][\"right_block_b_min\"]\n", "\n", "fig = plt.figure()\n", "#plt.rcParams[\"figure.figsize\"] = 20,20\n", "ax = Axes3D(fig)\n", "\n", "ax.set_xlim(0,255)\n", "ax.set_ylim(0,255)\n", "ax.set_zlim(0,255)\n", "ax.set_xlabel('H')\n", "ax.set_ylabel('S')\n", "ax.set_zlabel('V')\n", "ax.set_title('HSV colorspace OV right block max value')\n", "# ax.scatter(whiteBlock_R_zero, whiteBlock_G_zero, whiteBlock_B_zero,s = 15,c='y')\n", "ax.scatter(whiteBlock_R_one, whiteBlock_G_one, whiteBlock_B_one,s = 15,c='r')\n", "\n", "ax.scatter(whiteBlock_R_two, whiteBlock_G_two, whiteBlock_B_two,s = 15,c='g')\n", "ax.scatter(whiteBlock_R_three, whiteBlock_G_three, whiteBlock_B_three,s = 15,c='b')\n", "\n", "ax.scatter(whiteBlock_R_four, whiteBlock_G_four, whiteBlock_B_four,s = 15,c='y')\n", "ax.scatter(whiteBlock_R_five, whiteBlock_G_five, whiteBlock_B_five,s = 15,c='pink')\n", "ax.scatter(whiteBlock_R_six, whiteBlock_G_six, whiteBlock_B_six,s = 15,c='c')\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "data_all.columns" ] }, { "cell_type": "markdown", "metadata": { "collapsed": true }, "source": [ "hsv max min hist value h值要去掉" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 预处理数据" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Unnamed: 0left_block_Rleft_block_Gleft_block_Bleft_block_Hleft_block_Sleft_block_Vleft_block_lleft_block_aleft_block_b...right_grayValueright_grayStddevValueright_grayHistright_grayMaxright_grayMinwhite_grayValuewhite_grayStddevValuewhite_grayHistwhite_grayMaxwhite_grayMin
count82180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.000000...82180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.000000
mean8668.489827185.033086156.928292166.615296198.27820643.749002185.842529171.646495139.822962126.542602...164.57288917.293563162.055135197.940302133.605890205.9532120.185763206.318885207.747846205.103663
std9514.70185024.75595133.98158731.20899466.44750923.98009824.83650929.8238306.4003933.844306...17.7538854.44194125.11332615.38450823.20143316.0903210.46783116.57865215.99500116.190066
min0.00000043.00000030.00000033.0000000.0000000.00000043.00000034.000000121.000000111.000000...49.0000001.00000028.00000091.00000024.000000102.0000000.0000000.000000103.000000101.000000
25%1294.000000165.000000130.000000142.000000193.00000024.000000167.000000150.000000135.000000125.000000...152.00000014.000000144.000000190.000000115.000000197.0000000.000000197.000000199.000000195.000000
50%3917.500000187.000000158.000000169.000000227.00000041.000000187.000000173.000000140.000000127.000000...162.00000017.000000162.000000199.000000135.000000205.0000000.000000205.000000206.000000204.000000
75%14267.250000204.000000188.000000193.000000239.00000063.000000205.000000198.000000145.000000129.000000...179.00000021.000000180.000000208.000000152.000000218.0000000.000000218.000000219.000000217.000000
max36231.000000255.000000255.000000255.000000250.000000148.000000255.000000255.000000154.000000144.000000...248.00000041.000000251.000000255.000000242.000000255.00000017.000000254.000000255.000000255.000000
\n", "

8 rows × 151 columns

\n", "
" ], "text/plain": [ " Unnamed: 0 left_block_R left_block_G left_block_B left_block_H \\\n", "count 82180.000000 82180.000000 82180.000000 82180.000000 82180.000000 \n", "mean 8668.489827 185.033086 156.928292 166.615296 198.278206 \n", "std 9514.701850 24.755951 33.981587 31.208994 66.447509 \n", "min 0.000000 43.000000 30.000000 33.000000 0.000000 \n", "25% 1294.000000 165.000000 130.000000 142.000000 193.000000 \n", "50% 3917.500000 187.000000 158.000000 169.000000 227.000000 \n", "75% 14267.250000 204.000000 188.000000 193.000000 239.000000 \n", "max 36231.000000 255.000000 255.000000 255.000000 250.000000 \n", "\n", " left_block_S left_block_V left_block_l left_block_a left_block_b \\\n", "count 82180.000000 82180.000000 82180.000000 82180.000000 82180.000000 \n", "mean 43.749002 185.842529 171.646495 139.822962 126.542602 \n", "std 23.980098 24.836509 29.823830 6.400393 3.844306 \n", "min 0.000000 43.000000 34.000000 121.000000 111.000000 \n", "25% 24.000000 167.000000 150.000000 135.000000 125.000000 \n", "50% 41.000000 187.000000 173.000000 140.000000 127.000000 \n", "75% 63.000000 205.000000 198.000000 145.000000 129.000000 \n", "max 148.000000 255.000000 255.000000 154.000000 144.000000 \n", "\n", " ... right_grayValue right_grayStddevValue right_grayHist \\\n", "count ... 82180.000000 82180.000000 82180.000000 \n", "mean ... 164.572889 17.293563 162.055135 \n", "std ... 17.753885 4.441941 25.113326 \n", "min ... 49.000000 1.000000 28.000000 \n", "25% ... 152.000000 14.000000 144.000000 \n", "50% ... 162.000000 17.000000 162.000000 \n", "75% ... 179.000000 21.000000 180.000000 \n", "max ... 248.000000 41.000000 251.000000 \n", "\n", " right_grayMax right_grayMin white_grayValue white_grayStddevValue \\\n", "count 82180.000000 82180.000000 82180.000000 82180.000000 \n", "mean 197.940302 133.605890 205.953212 0.185763 \n", "std 15.384508 23.201433 16.090321 0.467831 \n", "min 91.000000 24.000000 102.000000 0.000000 \n", "25% 190.000000 115.000000 197.000000 0.000000 \n", "50% 199.000000 135.000000 205.000000 0.000000 \n", "75% 208.000000 152.000000 218.000000 0.000000 \n", "max 255.000000 242.000000 255.000000 17.000000 \n", "\n", " white_grayHist white_grayMax white_grayMin \n", "count 82180.000000 82180.000000 82180.000000 \n", "mean 206.318885 207.747846 205.103663 \n", "std 16.578652 15.995001 16.190066 \n", "min 0.000000 103.000000 101.000000 \n", "25% 197.000000 199.000000 195.000000 \n", "50% 205.000000 206.000000 204.000000 \n", "75% 218.000000 219.000000 217.000000 \n", "max 254.000000 255.000000 255.000000 \n", "\n", "[8 rows x 151 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "\n", "train_labels = data_all[\"index\"]\n", "train_features = data_all.drop(\"dateTime\",axis=1)\n", "train_features = train_features.drop(\"index\",axis=1)\n", "train_features = train_features.drop(\"whiteBalance\",axis=1)\n", "\n", "\n", "\n", "train_features.describe()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "#这一节的代码，不要执行\n", "\n", "train_features = train_features.drop(\"left_block_H\",axis=1)\n", "train_features = train_features.drop(\"left_block_S\",axis=1)\n", "train_features = train_features.drop(\"left_block_V\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H\",axis=1)\n", "train_features = train_features.drop(\"right_block_S\",axis=1)\n", "train_features = train_features.drop(\"right_block_V\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V\",axis=1)\n", "\n", "\n", "train_features = train_features.drop(\"left_block_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H_hist\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_hist\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_hist\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_hist\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_hist\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_hist\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H_min\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_min\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_min\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_min\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_min\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_min\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_min\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_min\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_min\",axis=1)\n", "\n", "\n", "\n", "train_features['lelf_right_R'] = train_features['left_block_R'] - train_features['right_block_R']\n", "train_features['lelf_right_G'] = train_features['left_block_G'] - train_features['right_block_G']\n", "train_features['lelf_right_B'] = train_features['left_block_B'] - train_features['right_block_B']\n", "\n", "# train_features['lelf_right_H'] = train_features['left_block_H'] - train_features['right_block_H']\n", "# train_features['lelf_right_S'] = train_features['left_block_S'] - train_features['right_block_S']\n", "# train_features['lelf_right_V'] = train_features['left_block_V'] - train_features['right_block_V']\n", "\n", "train_features['lelf_right_l'] = train_features['left_block_l'] - train_features['right_block_l']\n", "train_features['lelf_right_a'] = train_features['left_block_a'] - train_features['right_block_a']\n", "train_features['lelf_right_b'] = train_features['left_block_b'] - train_features['right_block_b']\n", "\n", "train_features['lelf_right_R_stddev'] = train_features['left_block_R_stddev'] - train_features['right_block_R_stddev']\n", "train_features['lelf_right_G_stddev'] = train_features['left_block_G_stddev'] - train_features['right_block_G_stddev']\n", "train_features['lelf_right_B_stddev'] = train_features['left_block_B_stddev'] - train_features['right_block_B_stddev']\n", "\n", "# train_features['lelf_right_H_stddev'] = train_features['left_block_H_stddev'] - train_features['right_block_H_stddev']\n", "# train_features['lelf_right_S_stddev'] = train_features['left_block_S_stddev'] - train_features['right_block_S_stddev']\n", "# train_features['lelf_right_V_stddev'] = train_features['left_block_V_stddev'] - train_features['right_block_V_stddev']\n", "\n", "train_features['lelf_right_l_stddev'] = train_features['left_block_l_stddev'] - train_features['right_block_l_stddev']\n", "train_features['lelf_right_a_stddev'] = train_features['left_block_a_stddev'] - train_features['right_block_a_stddev']\n", "train_features['lelf_right_b_stddev'] = train_features['left_block_b_stddev'] - train_features['right_block_b_stddev']\n", "\n", "train_features['lelf_right_R_hist'] = train_features['left_block_R_hist'] - train_features['right_block_R_hist']\n", "train_features['lelf_right_G_hist'] = train_features['left_block_G_hist'] - train_features['right_block_G_hist']\n", "train_features['lelf_right_B_hist'] = train_features['left_block_B_hist'] - train_features['right_block_B_hist']\n", "\n", "# train_features['lelf_right_H_hist'] = train_features['left_block_H_hist'] - train_features['right_block_H_hist']\n", "# train_features['lelf_right_S_hist'] = train_features['left_block_S_hist'] - train_features['right_block_S_hist']\n", "# train_features['lelf_right_V_hist'] = train_features['left_block_V_hist'] - train_features['right_block_V_hist']\n", "\n", "train_features['lelf_right_l_hist'] = train_features['left_block_l_hist'] - train_features['right_block_l_hist']\n", "train_features['lelf_right_a_hist'] = train_features['left_block_a_hist'] - train_features['right_block_a_hist']\n", "train_features['lelf_right_b_hist'] = train_features['left_block_b_hist'] - train_features['right_block_b_hist']\n", "\n", "train_features['lelf_right_R_max'] = train_features['left_block_R_max'] - train_features['right_block_R_max']\n", "train_features['lelf_right_G_max'] = train_features['left_block_G_max'] - train_features['right_block_G_max']\n", "train_features['lelf_right_B_max'] = train_features['left_block_B_max'] - train_features['right_block_B_max']\n", "\n", "# train_features['lelf_right_H_max'] = train_features['left_block_H_max'] - train_features['right_block_H_max']\n", "# train_features['lelf_right_S_max'] = train_features['left_block_S_max'] - train_features['right_block_S_max']\n", "# train_features['lelf_right_V_max'] = train_features['left_block_V_max'] - train_features['right_block_V_max']\n", "\n", "train_features['lelf_right_l_max'] = train_features['left_block_l_max'] - train_features['right_block_l_max']\n", "train_features['lelf_right_a_max'] = train_features['left_block_a_max'] - train_features['right_block_a_max']\n", "train_features['lelf_right_b_max'] = train_features['left_block_b_max'] - train_features['right_block_b_max']\n", "\n", "\n", "\n", "train_features['lelf_right_R_min'] = train_features['left_block_R_min'] - train_features['right_block_R_min']\n", "train_features['lelf_right_G_min'] = train_features['left_block_G_min'] - train_features['right_block_G_min']\n", "train_features['lelf_right_B_min'] = train_features['left_block_B_min'] - train_features['right_block_B_min']\n", "\n", "# train_features['lelf_right_H_min'] = train_features['left_block_H_min'] - train_features['right_block_H_min']\n", "# train_features['lelf_right_S_min'] = train_features['left_block_S_min'] - train_features['right_block_S_min']\n", "# train_features['lelf_right_V_min'] = train_features['left_block_V_min'] - train_features['right_block_V_min']\n", "\n", "train_features['lelf_right_l_min'] = train_features['left_block_l_min'] - train_features['right_block_l_min']\n", "train_features['lelf_right_a_min'] = train_features['left_block_a_min'] - train_features['right_block_a_min']\n", "train_features['lelf_right_b_min'] = train_features['left_block_b_min'] - train_features['right_block_b_min']\n", "\n", "train_features['lelf_right_gray_value'] = train_features['left_grayValue'] - train_features['right_grayValue']\n", "train_features['lelf_right_gray_stddev'] = train_features['left_grayStddevValue'] - train_features['right_grayStddevValue']\n", "train_features['lelf_right_gray_hist'] = train_features['left_grayHist'] - train_features['right_grayHist']\n", "train_features['lelf_right_gray_max'] = train_features['left_grayMax'] - train_features['right_grayMax']\n", "train_features['lelf_right_gray_min'] = train_features['left_grayMin'] - train_features['right_grayMin']\n", "train_features.describe()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# train_features_9是真正的训练数据" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
lelf_right_Rlelf_right_Glelf_right_Blelf_right_Hlelf_right_Slelf_right_Vlelf_right_llelf_right_alelf_right_blelf_right_R_stddev...lelf_right_S_minlelf_right_V_minlelf_right_l_minlelf_right_a_minlelf_right_b_minlelf_right_gray_valuelelf_right_gray_stddevlelf_right_gray_histlelf_right_gray_maxlelf_right_gray_min
count82180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.000000...82180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.00000082180.000000
mean-1.0481023.3309931.959504-18.906121-2.293879-0.7087611.831492-1.7066680.187296-0.147518...-5.165551-0.8480776.700134-0.8057560.2640911.860404-0.936396-6.5183860.1001586.950012
std18.39145730.14512824.27639956.81014721.81935019.02291825.6707796.0575601.4658088.414406...39.7427706.11053341.0047322.1758172.16213225.90742410.73210743.3506328.79753740.825874
min-59.000000-73.000000-66.000000-242.000000-82.000000-59.000000-68.000000-23.000000-4.000000-29.000000...-149.000000-24.000000-89.000000-10.000000-6.000000-67.000000-38.000000-251.000000-28.000000-85.000000
25%-16.000000-24.000000-20.000000-17.000000-20.000000-17.000000-21.000000-7.000000-1.000000-6.000000...-38.000000-5.000000-28.000000-2.000000-1.000000-21.000000-9.000000-39.000000-6.000000-28.000000
50%0.0000003.0000003.000000-2.000000-3.0000000.0000002.000000-1.0000000.000000-1.000000...-7.000000-1.0000008.000000-1.0000000.0000002.000000-2.000000-2.0000000.0000008.000000
75%12.00000027.00000020.0000003.00000017.00000012.00000021.0000004.0000001.0000007.000000...27.0000003.00000040.0000001.0000002.00000022.0000008.00000027.0000006.00000041.000000
max65.000000108.00000083.000000140.00000069.00000065.00000091.00000011.0000008.00000027.000000...144.00000029.000000151.0000007.00000011.00000093.00000032.000000141.00000045.000000150.000000
\n", "

8 rows × 50 columns

\n", "
" ], "text/plain": [ " lelf_right_R lelf_right_G lelf_right_B lelf_right_H lelf_right_S \\\n", "count 82180.000000 82180.000000 82180.000000 82180.000000 82180.000000 \n", "mean -1.048102 3.330993 1.959504 -18.906121 -2.293879 \n", "std 18.391457 30.145128 24.276399 56.810147 21.819350 \n", "min -59.000000 -73.000000 -66.000000 -242.000000 -82.000000 \n", "25% -16.000000 -24.000000 -20.000000 -17.000000 -20.000000 \n", "50% 0.000000 3.000000 3.000000 -2.000000 -3.000000 \n", "75% 12.000000 27.000000 20.000000 3.000000 17.000000 \n", "max 65.000000 108.000000 83.000000 140.000000 69.000000 \n", "\n", " lelf_right_V lelf_right_l lelf_right_a lelf_right_b \\\n", "count 82180.000000 82180.000000 82180.000000 82180.000000 \n", "mean -0.708761 1.831492 -1.706668 0.187296 \n", "std 19.022918 25.670779 6.057560 1.465808 \n", "min -59.000000 -68.000000 -23.000000 -4.000000 \n", "25% -17.000000 -21.000000 -7.000000 -1.000000 \n", "50% 0.000000 2.000000 -1.000000 0.000000 \n", "75% 12.000000 21.000000 4.000000 1.000000 \n", "max 65.000000 91.000000 11.000000 8.000000 \n", "\n", " lelf_right_R_stddev ... lelf_right_S_min lelf_right_V_min \\\n", "count 82180.000000 ... 82180.000000 82180.000000 \n", "mean -0.147518 ... -5.165551 -0.848077 \n", "std 8.414406 ... 39.742770 6.110533 \n", "min -29.000000 ... -149.000000 -24.000000 \n", "25% -6.000000 ... -38.000000 -5.000000 \n", "50% -1.000000 ... -7.000000 -1.000000 \n", "75% 7.000000 ... 27.000000 3.000000 \n", "max 27.000000 ... 144.000000 29.000000 \n", "\n", " lelf_right_l_min lelf_right_a_min lelf_right_b_min \\\n", "count 82180.000000 82180.000000 82180.000000 \n", "mean 6.700134 -0.805756 0.264091 \n", "std 41.004732 2.175817 2.162132 \n", "min -89.000000 -10.000000 -6.000000 \n", "25% -28.000000 -2.000000 -1.000000 \n", "50% 8.000000 -1.000000 0.000000 \n", "75% 40.000000 1.000000 2.000000 \n", "max 151.000000 7.000000 11.000000 \n", "\n", " lelf_right_gray_value lelf_right_gray_stddev lelf_right_gray_hist \\\n", "count 82180.000000 82180.000000 82180.000000 \n", "mean 1.860404 -0.936396 -6.518386 \n", "std 25.907424 10.732107 43.350632 \n", "min -67.000000 -38.000000 -251.000000 \n", "25% -21.000000 -9.000000 -39.000000 \n", "50% 2.000000 -2.000000 -2.000000 \n", "75% 22.000000 8.000000 27.000000 \n", "max 93.000000 32.000000 141.000000 \n", "\n", " lelf_right_gray_max lelf_right_gray_min \n", "count 82180.000000 82180.000000 \n", "mean 0.100158 6.950012 \n", "std 8.797537 40.825874 \n", "min -28.000000 -85.000000 \n", "25% -6.000000 -28.000000 \n", "50% 0.000000 8.000000 \n", "75% 6.000000 41.000000 \n", "max 45.000000 150.000000 \n", "\n", "[8 rows x 50 columns]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_features_9 = pd.DataFrame()\n", "train_features_9['lelf_right_R'] = train_features['left_block_R'] - train_features['right_block_R']\n", "train_features_9['lelf_right_G'] = train_features['left_block_G'] - train_features['right_block_G']\n", "train_features_9['lelf_right_B'] = train_features['left_block_B'] - train_features['right_block_B']\n", "\n", "train_features_9['lelf_right_H'] = train_features['left_block_H'] - train_features['right_block_H']\n", "train_features_9['lelf_right_S'] = train_features['left_block_S'] - train_features['right_block_S']\n", "train_features_9['lelf_right_V'] = train_features['left_block_V'] - train_features['right_block_V']\n", "\n", "train_features_9['lelf_right_l'] = train_features['left_block_l'] - train_features['right_block_l']\n", "train_features_9['lelf_right_a'] = train_features['left_block_a'] - train_features['right_block_a']\n", "train_features_9['lelf_right_b'] = train_features['left_block_b'] - train_features['right_block_b']\n", "\n", "train_features_9['lelf_right_R_stddev'] = train_features['left_block_R_stddev'] - train_features['right_block_R_stddev']\n", "train_features_9['lelf_right_G_stddev'] = train_features['left_block_G_stddev'] - train_features['right_block_G_stddev']\n", "train_features_9['lelf_right_B_stddev'] = train_features['left_block_B_stddev'] - train_features['right_block_B_stddev']\n", "\n", "train_features_9['lelf_right_H_stddev'] = train_features['left_block_H_stddev'] - train_features['right_block_H_stddev']\n", "train_features_9['lelf_right_S_stddev'] = train_features['left_block_S_stddev'] - train_features['right_block_S_stddev']\n", "train_features_9['lelf_right_V_stddev'] = train_features['left_block_V_stddev'] - train_features['right_block_V_stddev']\n", "\n", "train_features_9['lelf_right_l_stddev'] = train_features['left_block_l_stddev'] - train_features['right_block_l_stddev']\n", "train_features_9['lelf_right_a_stddev'] = train_features['left_block_a_stddev'] - train_features['right_block_a_stddev']\n", "train_features_9['lelf_right_b_stddev'] = train_features['left_block_b_stddev'] - train_features['right_block_b_stddev']\n", "\n", "train_features_9['lelf_right_R_hist'] = train_features['left_block_R_hist'] - train_features['right_block_R_hist']\n", "train_features_9['lelf_right_G_hist'] = train_features['left_block_G_hist'] - train_features['right_block_G_hist']\n", "train_features_9['lelf_right_B_hist'] = train_features['left_block_B_hist'] - train_features['right_block_B_hist']\n", "\n", "train_features_9['lelf_right_H_hist'] = train_features['left_block_H_hist'] - train_features['right_block_H_hist']\n", "train_features_9['lelf_right_S_hist'] = train_features['left_block_S_hist'] - train_features['right_block_S_hist']\n", "train_features_9['lelf_right_V_hist'] = train_features['left_block_V_hist'] - train_features['right_block_V_hist']\n", "\n", "train_features_9['lelf_right_l_hist'] = train_features['left_block_l_hist'] - train_features['right_block_l_hist']\n", "train_features_9['lelf_right_a_hist'] = train_features['left_block_a_hist'] - train_features['right_block_a_hist']\n", "train_features_9['lelf_right_b_hist'] = train_features['left_block_b_hist'] - train_features['right_block_b_hist']\n", "\n", "train_features_9['lelf_right_R_max'] = train_features['left_block_R_max'] - train_features['right_block_R_max']\n", "train_features_9['lelf_right_G_max'] = train_features['left_block_G_max'] - train_features['right_block_G_max']\n", "train_features_9['lelf_right_B_max'] = train_features['left_block_B_max'] - train_features['right_block_B_max']\n", "\n", "train_features_9['lelf_right_H_max'] = train_features['left_block_H_max'] - train_features['right_block_H_max']\n", "train_features_9['lelf_right_S_max'] = train_features['left_block_S_max'] - train_features['right_block_S_max']\n", "train_features_9['lelf_right_V_max'] = train_features['left_block_V_max'] - train_features['right_block_V_max']\n", "\n", "train_features_9['lelf_right_l_max'] = train_features['left_block_l_max'] - train_features['right_block_l_max']\n", "train_features_9['lelf_right_a_max'] = train_features['left_block_a_max'] - train_features['right_block_a_max']\n", "train_features_9['lelf_right_b_max'] = train_features['left_block_b_max'] - train_features['right_block_b_max']\n", "\n", "train_features_9['lelf_right_R_min'] = train_features['left_block_R_min'] - train_features['right_block_R_min']\n", "train_features_9['lelf_right_G_min'] = train_features['left_block_G_min'] - train_features['right_block_G_min']\n", "train_features_9['lelf_right_B_min'] = train_features['left_block_B_min'] - train_features['right_block_B_min']\n", "\n", "train_features_9['lelf_right_H_min'] = train_features['left_block_H_min'] - train_features['right_block_H_min']\n", "train_features_9['lelf_right_S_min'] = train_features['left_block_S_min'] - train_features['right_block_S_min']\n", "train_features_9['lelf_right_V_min'] = train_features['left_block_V_min'] - train_features['right_block_V_min']\n", "\n", "train_features_9['lelf_right_l_min'] = train_features['left_block_l_min'] - train_features['right_block_l_min']\n", "train_features_9['lelf_right_a_min'] = train_features['left_block_a_min'] - train_features['right_block_a_min']\n", "train_features_9['lelf_right_b_min'] = train_features['left_block_b_min'] - train_features['right_block_b_min']\n", "\n", "# train_features_9['left_grayValue']= train_features['left_grayValue'];\n", "# train_features_9['left_grayStddevValue']= train_features['left_grayStddevValue'];\n", "# train_features_9['left_grayHist']= train_features['left_grayHist'];\n", "# train_features_9['left_grayMax']= train_features['left_grayMax'];\n", "# train_features_9['left_grayMin']= train_features['left_grayMin'];\n", "\n", "# train_features_9['right_grayValue']= train_features['right_grayValue'];\n", "# train_features_9['right_grayStddevValue']= train_features['right_grayStddevValue'];\n", "# train_features_9['right_grayHist']= train_features['right_grayHist'];\n", "# train_features_9['right_grayMax']= train_features['right_grayMax'];\n", "# train_features_9['right_grayMin']= train_features['right_grayMin'];\n", "\n", "# train_features_9['lelf_R_stddev'] = train_features['left_block_R_stddev'] \n", "# train_features_9['lelf_G_stddev'] = train_features['left_block_G_stddev'] \n", "# train_features_9['lelf_B_stddev'] = train_features['left_block_B_stddev'] \n", "\n", "# train_features_9['left_block_R_min'] = train_features['left_block_R_min'] \n", "# train_features_9['left_block_G_min'] = train_features['left_block_G_min'] \n", "# train_features_9['left_block_B_min'] = train_features['left_block_B_min'] \n", "\n", "\n", "train_features_9['lelf_right_gray_value'] = train_features['left_grayValue'] - train_features['right_grayValue']\n", "train_features_9['lelf_right_gray_stddev'] = train_features['left_grayStddevValue'] - train_features['right_grayStddevValue']\n", "train_features_9['lelf_right_gray_hist'] = train_features['left_grayHist'] - train_features['right_grayHist']\n", "train_features_9['lelf_right_gray_max'] = train_features['left_grayMax'] - train_features['right_grayMax']\n", "train_features_9['lelf_right_gray_min'] = train_features['left_grayMin'] - train_features['right_grayMin']\n", "#train_features_9['index'] = train_labels\n", "train_features_9.describe()\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**去掉左边块的方差和白块和右边块的特征**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": false }, "outputs": [], "source": [ "# train_features = train_features.drop(\"left_block_R\",axis=1)\n", "# train_features = train_features.drop(\"left_block_G\",axis=1)\n", "# train_features = train_features.drop(\"left_block_B\",axis=1)\n", "\n", "# train_features = train_features.drop(\"left_block_R_hist\",axis=1)\n", "# train_features = train_features.drop(\"left_block_G_hist\",axis=1)\n", "# train_features = train_features.drop(\"left_block_B_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_R_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_G_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_B_stddev\",axis=1)\n", "\n", "# train_features = train_features.drop(\"left_block_R_max\",axis=1)\n", "# train_features = train_features.drop(\"left_block_G_max\",axis=1)\n", "# train_features = train_features.drop(\"left_block_B_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H\",axis=1)\n", "train_features = train_features.drop(\"left_block_S\",axis=1)\n", "train_features = train_features.drop(\"left_block_V\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H_hist\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_hist\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_l\",axis=1)\n", "train_features = train_features.drop(\"left_block_a\",axis=1)\n", "train_features = train_features.drop(\"left_block_b\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_l_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_a_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_b_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_l_hist\",axis=1)\n", "train_features = train_features.drop(\"left_block_a_hist\",axis=1)\n", "train_features = train_features.drop(\"left_block_b_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_l_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_a_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_b_max\",axis=1)\n", "##################################################################\n", "\n", "# train_features = train_features.drop(\"right_block_R\",axis=1)\n", "# train_features = train_features.drop(\"right_block_G\",axis=1)\n", "# train_features = train_features.drop(\"right_block_B\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_R_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_G_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_B_stddev\",axis=1)\n", "\n", "# train_features = train_features.drop(\"right_block_R_hist\",axis=1)\n", "# train_features = train_features.drop(\"right_block_G_hist\",axis=1)\n", "# train_features = train_features.drop(\"right_block_B_hist\",axis=1)\n", "\n", "# train_features = train_features.drop(\"right_block_R_max\",axis=1)\n", "# train_features = train_features.drop(\"right_block_G_max\",axis=1)\n", "# train_features = train_features.drop(\"right_block_B_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H\",axis=1)\n", "train_features = train_features.drop(\"right_block_S\",axis=1)\n", "train_features = train_features.drop(\"right_block_V\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_hist\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_hist\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_l\",axis=1)\n", "train_features = train_features.drop(\"right_block_a\",axis=1)\n", "train_features = train_features.drop(\"right_block_b\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_l_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_a_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_b_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_l_hist\",axis=1)\n", "train_features = train_features.drop(\"right_block_a_hist\",axis=1)\n", "train_features = train_features.drop(\"right_block_b_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_l_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_a_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_b_max\",axis=1)\n", "\n", "####################################################################\n", "\n", "train_features = train_features.drop(\"whiteBlock_R\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_G\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_B\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_R_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_G_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_B_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_R_hist\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_G_hist\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_B_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_R_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_G_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_B_max\",axis=1)\n", "\n", "\n", "train_features = train_features.drop(\"whiteBlock_H\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_hist\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_hist\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_max\",axis=1)\n", "\n", "\n", "\n", "train_features = train_features.drop(\"whiteBlock_l\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_a\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_b\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_l_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_a_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_b_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_l_hist\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_a_hist\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_b_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_l_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_a_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_b_max\",axis=1)\n", "\n", "##################################################################\n", "\n", "\n", "\n", "train_features.describe()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**去掉所有块的方差特征**" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "train_features = train_features.drop(\"left_block_R_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_G_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_B_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_l_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_a_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_b_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_R_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_G_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_B_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_l_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_a_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_b_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_R_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_G_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_B_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_l_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_a_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_b_stddev\",axis=1)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# train_features = train_features.drop(\"left_block_R\",axis=1)\n", "# train_features = train_features.drop(\"left_block_G\",axis=1)\n", "# train_features = train_features.drop(\"left_block_B\",axis=1)\n", "\n", "# train_features = train_features.drop(\"left_block_H\",axis=1)\n", "# train_features = train_features.drop(\"left_block_S\",axis=1)\n", "# train_features = train_features.drop(\"left_block_V\",axis=1)\n", "\n", "# train_features = train_features.drop(\"left_block_l\",axis=1)\n", "# train_features = train_features.drop(\"left_block_a\",axis=1)\n", "# train_features = train_features.drop(\"left_block_b\",axis=1)\n", "\n", "# train_features = train_features.drop(\"right_block_R\",axis=1)\n", "# train_features = train_features.drop(\"right_block_G\",axis=1)\n", "# train_features = train_features.drop(\"right_block_B\",axis=1)\n", "\n", "# train_features = train_features.drop(\"right_block_H\",axis=1)\n", "# train_features = train_features.drop(\"right_block_S\",axis=1)\n", "# train_features = train_features.drop(\"right_block_V\",axis=1)\n", "\n", "# train_features = train_features.drop(\"right_block_l\",axis=1)\n", "# train_features = train_features.drop(\"right_block_a\",axis=1)\n", "# train_features = train_features.drop(\"right_block_b\",axis=1)\n", "\n", "# train_features = train_features.drop(\"whiteBlock_R\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_G\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_B\",axis=1)\n", "\n", "# train_features = train_features.drop(\"whiteBlock_H\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_S\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_V\",axis=1)\n", "\n", "# train_features = train_features.drop(\"whiteBlock_l\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_a\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_b\",axis=1)\n", "\n", "# train_features = train_features.drop(\"left_block_R_hist\",axis=1)\n", "# train_features = train_features.drop(\"left_block_G_hist\",axis=1)\n", "# train_features = train_features.drop(\"left_block_B_hist\",axis=1)\n", "\n", "# train_features = train_features.drop(\"left_block_H_hist\",axis=1)\n", "# train_features = train_features.drop(\"left_block_S_hist\",axis=1)\n", "# train_features = train_features.drop(\"left_block_V_hist\",axis=1)\n", "\n", "# train_features = train_features.drop(\"left_block_l_hist\",axis=1)\n", "# train_features = train_features.drop(\"left_block_a_hist\",axis=1)\n", "# train_features = train_features.drop(\"left_block_b_hist\",axis=1)\n", "\n", "# train_features = train_features.drop(\"right_block_R_hist\",axis=1)\n", "# train_features = train_features.drop(\"right_block_G_hist\",axis=1)\n", "# train_features = train_features.drop(\"right_block_B_hist\",axis=1)\n", "\n", "# train_features = train_features.drop(\"right_block_H_hist\",axis=1)\n", "# train_features = train_features.drop(\"right_block_S_hist\",axis=1)\n", "# train_features = train_features.drop(\"right_block_V_hist\",axis=1)\n", "\n", "# train_features = train_features.drop(\"right_block_l_hist\",axis=1)\n", "# train_features = train_features.drop(\"right_block_a_hist\",axis=1)\n", "# train_features = train_features.drop(\"right_block_b_hist\",axis=1)\n", "\n", "# train_features = train_features.drop(\"whiteBlock_R_hist\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_G_hist\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_B_hist\",axis=1)\n", "\n", "# train_features = train_features.drop(\"whiteBlock_H_hist\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_S_hist\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_V_hist\",axis=1)\n", "\n", "# train_features = train_features.drop(\"whiteBlock_l_hist\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_a_hist\",axis=1)\n", "# train_features = train_features.drop(\"whiteBlock_b_hist\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_R_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_G_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_B_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_l_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_a_stddev\",axis=1)\n", "train_features = train_features.drop(\"left_block_b_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_R_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_G_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_B_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_l_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_a_stddev\",axis=1)\n", "train_features = train_features.drop(\"right_block_b_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_R_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_G_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_B_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_l_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_a_stddev\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_b_stddev\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_R_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_G_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_B_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_H_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_S_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_V_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"left_block_l_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_a_max\",axis=1)\n", "train_features = train_features.drop(\"left_block_b_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_R_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_G_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_B_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_H_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_S_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_V_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"right_block_l_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_a_max\",axis=1)\n", "train_features = train_features.drop(\"right_block_b_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_R_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_G_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_B_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_H_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_S_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_V_max\",axis=1)\n", "\n", "train_features = train_features.drop(\"whiteBlock_l_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_a_max\",axis=1)\n", "train_features = train_features.drop(\"whiteBlock_b_max\",axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "train_features.describe()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 测试一些算法" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "#test all those Ensemble Methods\n", "\n", "from sklearn.ensemble import AdaBoostClassifier\n", "from sklearn.ensemble import AdaBoostRegressor\n", "\n", "from sklearn.ensemble import BaggingClassifier\n", "from sklearn.ensemble import BaggingRegressor\n", "\n", "from sklearn.ensemble import ExtraTreesClassifier\n", "from sklearn.ensemble import ExtraTreesRegressor\n", "\n", "from sklearn.ensemble import GradientBoostingClassifier\n", "from sklearn.ensemble import GradientBoostingRegressor\n", "\n", "from sklearn.ensemble import IsolationForest\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.ensemble import RandomTreesEmbedding\n", "\n", "from sklearn.ensemble import StackingClassifier\n", "from sklearn.ensemble import StackingRegressor\n", "\n", "from sklearn.ensemble import VotingClassifier\n", "from sklearn.ensemble import VotingRegressor\n", "\n", "#from sklearn.ensemble import HistGradientBoostingRegressor\n", "#from sklearn.ensemble import HistGradientBoostingClassifier\n", "\n", "from sklearn.metrics import accuracy_score\n", "from sklearn.metrics import f1_score\n", "from sklearn.metrics import precision_score\n", "from sklearn.metrics import recall_score\n", "\n", "from sklearn.model_selection import train_test_split\n", "X_train ,X_test,y_train,y_test = train_test_split(train_features_9,train_labels,test_size = 0.3, random_state = 20)\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "accuracy_score : 0.9800032449095482\n", "f1_score : [0.99355631 0.96864764 0.98784441 0.96099719 0.98147301 0.90247074\n", " 0.97109283 0.98481287]\n", "precision_score: [0.99447174 0.97457627 0.98880597 0.95067621 0.97945675 0.99142857\n", " 0.96137738 0.98784911]\n", "recall_score : [0.99264255 0.9627907 0.98688472 0.97154472 0.98349759 0.82816229\n", " 0.98100665 0.98179524]\n", "runing time: 0:00:56.031340\n" ] } ], "source": [ "#case 1 AdaBoostClassifier \n", "#https://www.cnblogs.com/pinard/p/6136914.html\n", "from sklearn.tree import DecisionTreeClassifier\n", "from datetime import datetime\n", "bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=15, min_samples_split=20, min_samples_leaf=10),\n", " n_estimators=50, learning_rate=0.8)\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "bdt.fit(X_train, y_train)\n", "pred = bdt.predict(X_test)\n", "print (\"accuracy_score :\" , accuracy_score(y_test,pred))\n", "print (\"f1_score :\" , f1_score(y_test,pred,average=None))\n", "print (\"precision_score:\" , precision_score(y_test,pred,average=None))\n", "print (\"recall_score :\" , recall_score(y_test,pred,average=None))\n", "\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#case 2 AdaBoostRegressor\n", "#https://www.programcreek.com/python/example/86712/sklearn.ensemble.AdaBoostRegressor\n", "from sklearn.tree import DecisionTreeRegressor\n", "from datetime import datetime\n", "bdt = AdaBoostRegressor(DecisionTreeRegressor(max_depth=15, min_samples_split=20, min_samples_leaf=10),\n", " n_estimators=50, learning_rate=0.8)\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "bdt.fit(X_train, y_train)\n", "pred = bdt.predict(X_test)\n", "test_accuracy = bdt.score(X_test, y_test)\n", "print(test_accuracy)\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#case 3 BaggingClassifier \n", "from sklearn.tree import DecisionTreeClassifier\n", "from datetime import datetime\n", "bc = BaggingClassifier(DecisionTreeClassifier(max_depth=15, min_samples_split=20, min_samples_leaf=10),\n", " max_samples=0.5,max_features=0.5)\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "bc.fit(X_train, y_train)\n", "pred = bc.predict(X_test)\n", "print (\"accuracy_score :\" , accuracy_score(y_test,pred))\n", "print (\"f1_score :\" , f1_score(y_test,pred,average=None))\n", "print (\"precision_score:\" , precision_score(y_test,pred,average=None))\n", "print (\"recall_score :\" , recall_score(y_test,pred,average=None))\n", "\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#case 4 BaggingRegressor\n", "from sklearn.tree import DecisionTreeRegressor\n", "from datetime import datetime\n", "br = BaggingRegressor(DecisionTreeRegressor(max_depth=15, min_samples_split=20, min_samples_leaf=10),\n", " max_samples=0.5,max_features=0.5)\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "br.fit(X_train, y_train)\n", "pred = br.predict(X_test)\n", "print(bdt.score(X_test, y_test))\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n", "'''\n", "Bagging通过降低基分类器的方差，改善了泛化误差\n", "其性能依赖于基分类器的稳定性；如果基分类器不稳定，bagging有助于降低训练数据的随机波动导致的误差；如果稳定，则集成分类器的误差主要由基分类器的偏倚引起\n", "由于每个样本被选中的概率相同，因此bagging并不侧重于训练数据集中的任何特定实例\n", "'''" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#case 5 ExtraTreesClassifier \n", "from datetime import datetime\n", "ec = ExtraTreesClassifier(n_estimators=50,max_depth=20,min_samples_leaf=50)\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "ec.fit(X_train, y_train)\n", "pred = ec.predict(X_test)\n", "print (\"accuracy_score :\" , accuracy_score(y_test,pred))\n", "print (\"f1_score :\" , f1_score(y_test,pred,average=None))\n", "print (\"precision_score:\" , precision_score(y_test,pred,average=None))\n", "print (\"recall_score :\" , recall_score(y_test,pred,average=None))\n", "\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "#case 6 ExtraTreesRegressor \n", "from datetime import datetime\n", "er = ExtraTreesRegressor(n_estimators=50,max_depth=20,min_samples_leaf=50)\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "er.fit(X_train, y_train)\n", "pred = er.predict(X_test)\n", "print (\"score :\" , er.score(X_test, y_test))\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#case 7 RandomForestClassifier \n", "from datetime import datetime\n", "rfc = RandomForestClassifier(n_estimators=50,max_depth=20,min_samples_leaf=50)\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "rfc.fit(X_train, y_train)\n", "pred = rfc.predict(X_test)\n", "print (\"accuracy_score :\" , accuracy_score(y_test,pred))\n", "print (\"f1_score :\" , f1_score(y_test,pred,average=None))\n", "print (\"precision_score:\" , precision_score(y_test,pred,average=None))\n", "print (\"recall_score :\" , recall_score(y_test,pred,average=None))\n", "\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "#case 8 RandomForestRegressor \n", "from datetime import datetime\n", "rfr = RandomForestRegressor(n_estimators=50,max_depth=20,min_samples_leaf=50)\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "rfr.fit(X_train, y_train)\n", "print (\"score :\" , rfr.score(X_test, y_test))\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#case 9 C-Support Vector Classification.\n", "from sklearn.svm import SVC\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "svc_linear = SVC(kernel = 'linear',C=0.1)\n", "#svm linear accuracy score: 0.974885004599816\n", "svc_linear.fit(X_train, y_train)\n", "# pred = clf_svm_linear.predict(X_test)\n", "# print \"svm linear accuracy score:\" , accuracy_score(y_test,pred)\n", "# print \"f1 score:\" , f1_score(y_test,pred,average='micro')\n", "pred = svc_linear.predict(X_test)\n", "print (\"svm linear accuracy score:\" , accuracy_score(y_test,pred))\n", "print (\"f1 score :\" , f1_score(y_test,pred,average=None))\n", "print (\"precision_score:\" , precision_score(y_test,pred,average=None))\n", "print (\"recall_score :\" , recall_score(y_test,pred,average=None))\n", "#print(\"preds:\",pred[:10])\n", "#print('trues:\\n',y_test[:10])\n", "#print(\"\\n\")\n", "\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n", "\n", "###针对同一份数据，\n", "#svc_poly = SVC(kernel='poly',degree=3,gamma=0.001,C=0.1)\n", "#clf_svc_poly = SVC(kernel='poly',degree=3,gamma=0.00001,C=0.1)\n", "\n", "##svm polynomial accuracy score: 0.37460901563937443\n", "#svc_poly.fit(X_train, y_train)\n", "#pred_poly = svc_poly.predict(X_test)\n", "#print (\"svm polynomial accuracy score:\" , accuracy_score(y_test,pred_poly))\n", "#print (\"f1 score :\" , f1_score(y_test,pred_poly,average=None))\n", "#print (\"precision_score:\" , precision_score(y_test,pred_poly,average=None))\n", "#print (\"recall_score :\" , recall_score(y_test,pred_poly,average=None))\n", "\n", "#svc_rbf = SVC(kernel='rbf', gamma=0.05,C=0.1)\n", "##svm rbf accuracy score: 0.284360625574977\n", "#svc_rbf.fit(X_train, y_train)\n", "#pred_rbf = svc_rbf.predict(X_test)\n", "#print (\"svm rbf accuracy score:\" , accuracy_score(y_test,pred_rbf))\n", "#print (\"f1 score :\" , f1_score(y_test,pred_rbf,average=None))\n", "#print (\"precision_score:\" , precision_score(y_test,pred_rbf,average=None))\n", "#print (\"recall_score :\" , recall_score(y_test,pred_rbf,average=None))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#case 9 C-Support Vector Classification.\n", "from sklearn.svm import SVR\n", "\n", "trarining_start_time = datetime.now()\n", "\n", "svr_linear = SVR(kernel = 'linear',C=0.1)\n", "#svm linear accuracy score: 0.974885004599816\n", "svr_linear.fit(X_train, y_train)\n", "# pred = clf_svm_linear.predict(X_test)\n", "# print \"svm linear accuracy score:\" , accuracy_score(y_test,pred)\n", "# print \"f1 score:\" , f1_score(y_test,pred,average='micro')\n", "pred = svr_linear.predict(X_test)\n", "print (\"svm linear accuracy score:\" , accuracy_score(y_test,pred))\n", "print (\"f1 score :\" , f1_score(y_test,pred,average=None))\n", "print (\"precision_score:\" , precision_score(y_test,pred,average=None))\n", "print (\"recall_score :\" , recall_score(y_test,pred,average=None))\n", "print(\"preds:\",pred[:10])\n", "print('trues:\\n',y_test[:10])\n", "print(\"\\n\")\n", "\n", "training_stop_time = datetime.now()\n", "print (\"runing time:\",(training_stop_time - trarining_start_time))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# svc支持向量机算法" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#from sklearn.model_selection import KFold\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.metrics import accuracy_score\n", "from sklearn.svm import SVC\n", "from sklearn.metrics import f1_score\n", "from sklearn.metrics import precision_score\n", "from sklearn.metrics import recall_score\n", "#集成学习(Ensemble Learning) \n", "from sklearn.ensemble import ExtraTreesClassifier\n", "from sklearn.ensemble import AdaBoostClassifier\n", "\n", "#报错：ModuleNotFoundError: No module named 'sklearn.cross_validation'\n", "#原因：当前 sklearn 版本中 cross_validation 已经替换成了 model_selection，但其中的函数功能并没有变化\n", "#from sklearn.cross_validation import train_test_split\n", "from sklearn.model_selection import train_test_split\n", "X_train ,X_test,y_train,y_test = train_test_split(train_features_9,train_labels,test_size = 0.3, random_state = 20)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "##fit_transform,fit,transform区别和作用详解\n", "###fit和transform没有任何关系，仅仅是数据处理的两个不同环节，之所以出来fit_transform这个函数名，仅仅是为了写代码方便，会高效一点。\n", "###sklearn里的封装好的各种算法使用前都要fit，fit相对于整个代码而言，为后续API服务。fit之后，然后调用各种API方法,transform只是其中一个API方法，所以当你调用transform之外的方法，也必须要先fit。\n", "###fit原义指的是安装、使适合的意思，其实有点train的含义，但是和train不同的是，它并不是一个训练的过程，而是一个适配的过程，过程都是确定的，最后得到一个可用于转换的有价值的信息。\n", "###https://blog.csdn.net/weixin_38278334/article/details/82971752\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 逻辑回归算法测试" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.linear_model import LogisticRegression\n", "clf = LogisticRegression(random_state=0).fit(X_train, y_train)\n", "clf.predict(X_test)\n", "clf.score(X_test, y_test)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 特征/模型选择" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "##这个不需要了！\n", "X = train_features.values\n", "y = train_labels.values\n", "\n", "kf = KFold(n_splits=5)\n", "kf.get_n_splits(X)\n", "\n", "print(kf) \n", "\n", "for train_index, test_index in kf.split(X):\n", " print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n", " X_train, X_test = X[train_index], X[test_index]\n", " y_train, y_test = y[train_index], y[test_index]\n", " \n", " \n", " from datetime import datetime\n", " trarining_start_time = datetime.now()\n", "\n", " clf_svm_linear = SVC(kernel = 'linear',gamma=0.00001,C=0.1)\n", " clf_svm_linear = clf_svm_linear.fit(X_train, y_train)\n", " pred = clf_svm_linear.predict(X_test)\n", " print \"svm linear accuracy score:\" , accuracy_score(y_test,pred)\n", " print \"f1 score:\" , f1_score(y_test,pred,average='micro')\n", "\n", "\n", " training_stop_time = datetime.now()\n", "\n", " print \"runing time:\",(training_stop_time - trarining_start_time)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import StratifiedKFold #交叉验证\n", "from sklearn.model_selection import GridSearchCV #网格搜索\n", "from sklearn.model_selection import train_test_split #将数据集分开成训练集和测试集\n", "from xgboost import XGBClassifier #xgboost\n", "\n", "#这个不需要了！\n", "model = XGBClassifier()\n", "learning_rate = [0.0001,0.001,0.01,0.1,0.2,0.3] #学习率\n", "gamma = [1, 0.1, 0.01, 0.001]\n", "param_grid = dict(learning_rate = learning_rate,gamma = gamma)#转化为字典格式，网络搜索要求\n", "kflod = StratifiedKFold(n_splits=10, shuffle = True,random_state=7)#将训练/测试数据集划分10个互斥子集，\n", "grid_search = GridSearchCV(model,param_grid,scoring = 'neg_log_loss',n_jobs = -1,cv = kflod)\n", "#scoring指定损失函数类型，n_jobs指定全部cpu跑，cv指定交叉验证\n", "grid_result = grid_search.fit(X_train, y_train) #运行网格搜索\n", "print(\"Best: %f using %s\" % (grid_result.best_score_,grid_search.best_params_))\n", "#grid_scores_：给出不同参数情况下的评价结果。best_params_：描述了已取得最佳结果的参数的组合\n", "#best_score_：成员提供优化过程期间观察到的最好的评分\n", "#具有键作为列标题和值作为列的dict，可以导入到DataFrame中。\n", "#注意，“params”键用于存储所有参数候选项的参数设置列表。\n", "means = grid_result.cv_results_['mean_test_score']\n", "params = grid_result.cv_results_['params']\n", "for mean,param in zip(means,params):\n", " print(\"%f with: %r\" % (mean,param))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import StratifiedKFold #交叉验证\n", "from sklearn.model_selection import GridSearchCV #网格搜索\n", "from sklearn.model_selection import GridSearchCV\n", "from sklearn import metrics \n", "#https://blog.csdn.net/WJWFighting/article/details/80983022\n", "thresholds=np.linspace(0,0.1,20)#设置gamma参数列表,生成等差数列\n", "thresholds\n", "param_grid={'gamma':thresholds}\n", "clf=GridSearchCV(SVC(kernel='rbf'),param_grid,cv=5)\n", "clf.fit(X_train, y_train)\n", "\n", "print(\"best param: {0}\\nbest score: {1}\".format(clf.best_params_, clf.best_score_))\n", "\n", "y_pred = clf.predict(X_test)\n", "\n", "print(\"查准率：\",metrics.precision_score(y_pred, y_test))\n", "print(\"召回率：\",metrics.recall_score(y_pred, y_test))\n", "print(\"F1：\",metrics.f1_score(y_pred, y_test))\n", "\n", "print(\"最佳效果：%0.3f\"% clf.best_score_)\n", "print(\"最优参数组合：\")\n", "best_parameters=clf.best_estimator_.get_params()\n", "for param_name in sorted(param_grid.keys()):\n", " print('\\t%s:%r' %(param_name,best_parameters[param_name]))\n", "\n", "#print(\"训练集评分：\",clf.score(x_train,y_train))\n", "#print(\"测试集评分：\",clf.score(x_test,y_test))\n", "\n", "\"\"\"\n", "SVC方法。常用的参数如下：\n", "C：默认为1.0，是对于错误的惩罚项。\n", "kernel：指定算法的核函数，默认为'rbf'，常用的有'linear'，'poly'，'rbf'，'sigmoid'，'precomputed'。\n", "degree：多项式核函数的次数（'poly'），默认为3。 其他核函数会将其忽略。\n", "gamma：'rbf'，'poly'和'sigmoid'的核系数。 如果gamma是'auto'，那么将使用1 / n_features。\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "#X_train = train_features_9\n", "#y_train = train_labels\n", "# X_test = test_features\n", "# y_test = test_labels\n", "#clf_svm_linear = SVC(kernel = 'linear',gamma=0.00001,C=0.01)\n", "#clf_svm_linear = SVC(kernel = 'linear',gamma=0.01,C=0.01)\n", "#svm linear accuracy score: 0.9746101835242169\n", "clf_svm_linear = SVC(kernel = 'linear',C=0.1)\n", "#svm linear accuracy score: 0.974885004599816\n", "clf_svm_linear = clf_svm_linear.fit(X_train, y_train)\n", "# pred = clf_svm_linear.predict(X_test)\n", "# print \"svm linear accuracy score:\" , accuracy_score(y_test,pred)\n", "# print \"f1 score:\" , f1_score(y_test,pred,average='micro')\n", "pred = clf_svm_linear.predict(X_test)\n", "print (\"svm linear accuracy score:\" , accuracy_score(y_test,pred))\n", "print (\"f1 score :\" , f1_score(y_test,pred,average=None))\n", "print (\"precision_score:\" , precision_score(y_test,pred,average=None))\n", "print (\"recall_score :\" , recall_score(y_test,pred,average=None))\n", "print(\"preds:\",pred[:10])\n", "print('trues:\\n',y_test[:10])\n", "print(\"\\n\")\n", "###针对同一份数据，\n", "clf_svc_poly = SVC(kernel='poly',degree=3,gamma=0.001,C=0.1)\n", "#clf_svc_poly = SVC(kernel='poly',degree=3,gamma=0.00001,C=0.1)\n", "\n", "##svm polynomial accuracy score: 0.37460901563937443\n", "clf_svc_poly.fit(X_train, y_train)\n", "pred_poly = clf_svc_poly.predict(X_test)\n", "print (\"svm polynomial accuracy score:\" , accuracy_score(y_test,pred_poly))\n", "print (\"f1 score :\" , f1_score(y_test,pred_poly,average=None))\n", "print (\"precision_score:\" , precision_score(y_test,pred_poly,average=None))\n", "print (\"recall_score :\" , recall_score(y_test,pred_poly,average=None))\n", "\n", "clf_svc_rbf = SVC(kernel='rbf', gamma=0.05,C=0.1)\n", "##svm rbf accuracy score: 0.284360625574977\n", "clf_svc_rbf.fit(X_train, y_train)\n", "pred_rbf = clf_svc_rbf.predict(X_test)\n", "print (\"svm rbf accuracy score:\" , accuracy_score(y_test,pred_rbf))\n", "print (\"f1 score :\" , f1_score(y_test,pred_rbf,average=None))\n", "print (\"precision_score:\" , precision_score(y_test,pred_rbf,average=None))\n", "print (\"recall_score :\" , recall_score(y_test,pred_rbf,average=None))\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn_porter import Porter\n", "\n", "porter_clf_svm_linear = Porter(clf_svm_linear, language='c').export()\n", "#porter_clf_svm_poly = Porter(clf_svm_poly, language='c').export()\n", "# porter_clf_forest = Porter(clf_randomForest, language='c').export()\n", "#porter_clf_extra_forest = Porter(clf_extra_forest, language='c').export()\n", "\n", "#print(porter_clf_svm_linear)\n", "f = open(\"clf/clf_svm_linear_50features_2020.cpp\",'wb')\n", "#f = open(\"clf/clf_svm_linear_50features_20171207.txt\",'wb')\n", "#f = open(\"clf_svm_linear_125100_low_feature_data.txt\",'wb')\n", "f.write(porter_clf_svm_linear.encode())\n", "f.close()\n", "#f = open(\"clf_svm_poly_2457100_data.txt\",'wb')\n", "#f.write(porter_clf_svm_poly)\n", "#f.close()\n", "# f = open(\"clf/clf_randomForest_27features_stddev_c_0_01.txt\",'wb')\n", "# f.write(porter_clf_forest)\n", "# f.close()\n", "# f = open(\"oclf_extra_forest_2457100_data_0824.txt\",'wb')\n", "# f.write(porter_clf_extra_forest)\n", "# f.close()\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 随机森林算法" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#from sklearn.model_selection import KFold\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.metrics import accuracy_score\n", "from sklearn.svm import SVC\n", "from sklearn.metrics import f1_score\n", "from sklearn.metrics import precision_score\n", "from sklearn.metrics import recall_score\n", "\n", "\n", "from sklearn.ensemble import ExtraTreesClassifier\n", "from sklearn.ensemble import AdaBoostClassifier\n", "\n", "from sklearn.model_selection import train_test_split\n", "X_train ,X_test,y_train,y_test = train_test_split(train_features_9,train_labels,test_size = 0.3, random_state = 20)\n", "#X_train ,X_test,y_train,y_test = train_test_split(train_features_9,train_labels,test_size = 0.2, random_state = 20)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.metrics import classification_report\n", "from sklearn.metrics import classification_report,confusion_matrix\n", "from sklearn.metrics import f1_score\n", "from sklearn.metrics import precision_score\n", "from sklearn.metrics import recall_score\n", "\n", "#rfc = RandomForestClassifier(n_estimators=600)\n", "\n", "#rfc = RandomForestClassifier(n_estimators=50)\n", "#RandomForest accuracy score: 0.9955591300090916\n", "\n", "#rfc = RandomForestClassifier(n_estimators=50,min_samples_leaf=20)\n", "#RandomForest accuracy score: 0.9772012028813204/0.9803133086229806/0.9811874956290649/0.9852786908175397\n", "#50- 0.9648738541413158\n", "\n", "#rfc = RandomForestClassifier(n_estimators=50)\n", "#---- 0.9857629593575079\n", "#RandomForest accuracy score: 0.9955940974893349\n", "\n", "rfc = RandomForestClassifier(n_estimators=50,min_samples_leaf=50)\n", "#30-0.9595197533868743 40 - 0.9553013709742841 50 - 0.9528271274438225\n", "\n", "#rfc = RandomForestClassifier(n_estimators=100,min_samples_leaf=50)\n", "#RandomForest accuracy score: 0.97688649555913\n", "\n", "#rfc = RandomForestClassifier(n_estimators=50,min_samples_leaf=100)\n", "#RandomForest accuracy score: 0.9669906986502552\n", "\n", "\n", "rfc.fit(X_train, y_train)\n", "rfc_pred = rfc.predict(X_test)\n", "cr = classification_report(y_test,rfc_pred)\n", "print(cr)\n", "cm = confusion_matrix(y_test,rfc_pred)\n", "print(cm)\n", "\n", "print(\"---------------------------------\\n\")\n", "print (\"Accuracy of prediction:\",round((cm[0,0]+cm[1,1])/cm.sum(),3))\n", "print (\"RandomForest accuracy score:\" , accuracy_score(y_test,rfc_pred))\n", "print(\"---------------------------------\\n\")\n", "print (\"f1 score:\" , f1_score(y_test,rfc_pred,average='micro'))\n", "print (\"precision_score:\" , precision_score(y_test,rfc_pred,average='micro'))\n", "print (\"recall_score:\" , recall_score(y_test,rfc_pred,average='micro'))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn_porter import Porter\n", "\n", "porter_clf_rfc = Porter(rfc, language='c').export()\n", "#porter_clf_svm_poly = Porter(clf_svm_poly, language='c').export()\n", "# porter_clf_forest = Porter(clf_randomForest, language='c').export()\n", "#porter_clf_extra_forest = Porter(clf_extra_forest, language='c').export()\n", "\n", "#print(porter_clf_svm_linear)\n", "f = open(\"clf/ov_rtree50_f50_2020515.cpp\",'wb')\n", "#f = open(\"clf/clf_svm_linear_50features_20171207.txt\",'wb')\n", "#f = open(\"clf_svm_linear_125100_low_feature_data.txt\",'wb')\n", "f.write(porter_clf_rfc.encode())\n", "f.close()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import KFold\n", "\n", "X = train_features_9.values\n", "y = train_labels.values\n", "\n", "kf = KFold(n_splits=5)\n", "kf.get_n_splits(X)\n", "\n", "print(kf) \n", "\n", "for train_index, test_index in kf.split(X):\n", " print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n", " X_train, X_test = X[train_index], X[test_index]\n", " y_train, y_test = y[train_index], y[test_index]\n", " \n", " \n", " from datetime import datetime\n", " trarining_start_time = datetime.now()\n", "\n", " rfc = RandomForestClassifier(n_estimators=600)\n", " rfc.fit(X_train, y_train)\n", " rfc_pred = rfc.predict(X_test) \n", " print (\"svm linear accuracy score:\" , accuracy_score(y_test,rfc_pred))\n", " print (\"f1 score:\" , f1_score(y_test,rfc_pred,average='micro'))\n", " print (\"precision_score:\" , precision_score(y_test,rfc_pred,average='micro'))\n", " print (\"recall_score:\" , recall_score(y_test,rfc_pred,average='micro'))\n", "\n", " training_stop_time = datetime.now()\n", "\n", " print (\"runing time:\",(training_stop_time - trarining_start_time))\n", " print(\"\\n\\n\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "nsimu = 21\n", "accuracy=[0]*nsimu\n", "ntree = [0]*nsimu\n", "for i in range(1,nsimu):\n", " rfc = RandomForestClassifier(n_estimators=i*5,min_samples_split=10,max_depth=None,criterion='gini')\n", " rfc.fit(X_train, y_train)\n", " rfc_pred = rfc.predict(X_test)\n", " cm = confusion_matrix(y_test,rfc_pred)\n", " accuracy[i] = (cm[0,0]+cm[1,1])/cm.sum()\n", " ntree[i]=i*5\n", "\n", " print (\"RandomForest accuracy score:\" , accuracy_score(y_test,rfc_pred))\n", " print (\"f1 score:\" , f1_score(y_test,rfc_pred,average='micro')) \n", " print (\"Accuracy of prediction:\",round((cm[0,0]+cm[1,1])/cm.sum(),3))\n", "\n", " \n", "plt.figure(figsize=(10,6))\n", "plt.scatter(x=ntree[1:nsimu],y=accuracy[1:nsimu],s=60,c='red')\n", "plt.title(\"Number of trees in the Random Forest vs. prediction accuracy (criterion: 'gini')\", fontsize=18)\n", "plt.xlabel(\"Number of trees\", fontsize=15)\n", "plt.ylabel(\"Prediction accuracy from confusion matrix\", fontsize=15)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "from sklearn.utils import shuffle\n", "\n", "\n", "# data_shuffle1 = shuffle(data1)\n", "# #data_shuffle = data_all;\n", "# test_labels = data_shuffle1[\"index\"]\n", "# test_features = data_shuffle1.drop(\"dateTime\",axis=1)\n", "# test_features = test_features.drop(\"index\",axis=1)\n", "# test_features = test_features.drop(\"whiteBalance\",axis=1)\n", "\n", "\n", "# test_features = test_features.drop(\"left_block_R_stddev\",axis=1)\n", "# test_features = test_features.drop(\"left_block_G_stddev\",axis=1)\n", "# test_features = test_features.drop(\"left_block_B_stddev\",axis=1)\n", "\n", "# test_features = test_features.drop(\"left_block_H_stddev\",axis=1)\n", "# test_features = test_features.drop(\"left_block_S_stddev\",axis=1)\n", "# test_features = test_features.drop(\"left_block_V_stddev\",axis=1)\n", "\n", "# test_features = test_features.drop(\"left_block_l_stddev\",axis=1)\n", "# test_features = test_features.drop(\"left_block_a_stddev\",axis=1)\n", "# test_features = test_features.drop(\"left_block_b_stddev\",axis=1)\n", "\n", "# test_features = test_features.drop(\"right_block_R_stddev\",axis=1)\n", "# test_features = test_features.drop(\"right_block_G_stddev\",axis=1)\n", "# test_features = test_features.drop(\"right_block_B_stddev\",axis=1)\n", "\n", "# test_features = test_features.drop(\"right_block_H_stddev\",axis=1)\n", "# test_features = test_features.drop(\"right_block_S_stddev\",axis=1)\n", "# test_features = test_features.drop(\"right_block_V_stddev\",axis=1)\n", "\n", "# test_features = test_features.drop(\"right_block_l_stddev\",axis=1)\n", "# test_features = test_features.drop(\"right_block_a_stddev\",axis=1)\n", "# test_features = test_features.drop(\"right_block_b_stddev\",axis=1)\n", "\n", "# test_features = test_features.drop(\"whiteBlock_R_stddev\",axis=1)\n", "# test_features = test_features.drop(\"whiteBlock_G_stddev\",axis=1)\n", "# test_features = test_features.drop(\"whiteBlock_B_stddev\",axis=1)\n", "\n", "# test_features = test_features.drop(\"whiteBlock_H_stddev\",axis=1)\n", "# test_features = test_features.drop(\"whiteBlock_S_stddev\",axis=1)\n", "# test_features = test_features.drop(\"whiteBlock_V_stddev\",axis=1)\n", "\n", "# test_features = test_features.drop(\"whiteBlock_l_stddev\",axis=1)\n", "# test_features = test_features.drop(\"whiteBlock_a_stddev\",axis=1)\n", "# test_features = test_features.drop(\"whiteBlock_b_stddev\",axis=1)\n", "\n", "train_features_10 = pd.DataFrame()\n", "train_features_10['lelf_right_R'] = test_features['left_block_R'] - test_features['right_block_R']\n", "train_features_10['lelf_right_G'] = test_features['left_block_G'] - test_features['right_block_G']\n", "train_features_10['lelf_right_B'] = test_features['left_block_B'] - test_features['right_block_B']\n", "\n", "train_features_10['lelf_right_H'] = test_features['left_block_H'] - test_features['right_block_H']\n", "# train_features_10['lelf_right_S'] = test_features['left_block_S'] - test_features['right_block_S']\n", "train_features_10['lelf_right_V'] = test_features['left_block_V'] - test_features['right_block_V']\n", "\n", "train_features_10['lelf_right_l'] = test_features['left_block_l'] - test_features['right_block_l']\n", "train_features_10['lelf_right_a'] = test_features['left_block_a'] - test_features['right_block_a']\n", "train_features_10['lelf_right_b'] = test_features['left_block_b'] - test_features['right_block_b']\n", "\n", "train_features_10['lelf_right_R_stddev'] = test_features['left_block_R_stddev'] - test_features['right_block_R_stddev']\n", "train_features_10['lelf_right_G_stddev'] = test_features['left_block_G_stddev'] - test_features['right_block_G_stddev']\n", "train_features_10['lelf_right_B_stddev'] = test_features['left_block_B_stddev'] - test_features['right_block_B_stddev']\n", "\n", "train_features_10['lelf_right_H_stddev'] = test_features['left_block_H_stddev'] - test_features['right_block_H_stddev']\n", "# train_features_10['lelf_right_S_stddev'] = test_features['left_block_S_stddev'] - test_features['right_block_S_stddev']\n", "train_features_10['lelf_right_V_stddev'] = test_features['left_block_V_stddev'] - test_features['right_block_V_stddev']\n", "\n", "train_features_10['lelf_right_l_stddev'] = test_features['left_block_l_stddev'] - test_features['right_block_l_stddev']\n", "train_features_10['lelf_right_a_stddev'] = test_features['left_block_a_stddev'] - test_features['right_block_a_stddev']\n", "train_features_10['lelf_right_b_stddev'] = test_features['left_block_b_stddev'] - test_features['right_block_b_stddev']\n", "\n", "train_features_10['lelf_right_R_hist'] = test_features['left_block_R_hist'] - test_features['right_block_R_hist']\n", "train_features_10['lelf_right_G_hist'] = test_features['left_block_G_hist'] - test_features['right_block_G_hist']\n", "train_features_10['lelf_right_B_hist'] = test_features['left_block_B_hist'] - test_features['right_block_B_hist']\n", "\n", "train_features_10['lelf_right_H_hist'] = test_features['left_block_H_hist'] - test_features['right_block_H_hist']\n", "# train_features_10['lelf_right_S_hist'] = test_features['left_block_S_hist'] - test_features['right_block_S_hist']\n", "train_features_10['lelf_right_V_hist'] = test_features['left_block_V_hist'] - test_features['right_block_V_hist']\n", "\n", "train_features_10['lelf_right_l_hist'] = test_features['left_block_l_hist'] - test_features['right_block_l_hist']\n", "train_features_10['lelf_right_a_hist'] = test_features['left_block_a_hist'] - test_features['right_block_a_hist']\n", "train_features_10['lelf_right_b_hist'] = test_features['left_block_b_hist'] - test_features['right_block_b_hist']\n", "\n", "train_features_10['lelf_right_R_max'] = test_features['left_block_R_max'] - test_features['right_block_R_max']\n", "train_features_10['lelf_right_G_max'] = test_features['left_block_G_max'] - test_features['right_block_G_max']\n", "train_features_10['lelf_right_B_max'] = test_features['left_block_B_max'] - test_features['right_block_B_max']\n", "\n", "train_features_10['lelf_right_H_max'] = test_features['left_block_H_max'] - test_features['right_block_H_max']\n", "# train_features_10['lelf_right_S_max'] = test_features['left_block_S_max'] - test_features['right_block_S_max']\n", "train_features_10['lelf_right_V_max'] = test_features['left_block_V_max'] - test_features['right_block_V_max']\n", "\n", "train_features_10['lelf_right_l_max'] = test_features['left_block_l_max'] - test_features['right_block_l_max']\n", "train_features_10['lelf_right_a_max'] = test_features['left_block_a_max'] - test_features['right_block_a_max']\n", "train_features_10['lelf_right_b_max'] = test_features['left_block_b_max'] - test_features['right_block_b_max']\n", "\n", "\n", "train_features_10['lelf_right_R_min'] = test_features['left_block_R_min'] - test_features['right_block_R_min']\n", "train_features_10['lelf_right_G_min'] = test_features['left_block_G_min'] - test_features['right_block_G_min']\n", "train_features_10['lelf_right_B_min'] = test_features['left_block_B_min'] - test_features['right_block_B_min']\n", "\n", "train_features_10['lelf_right_H_min'] = test_features['left_block_H_min'] - test_features['right_block_H_min']\n", "# train_features_10['lelf_right_S_min'] = test_features['left_block_S_min'] - test_features['right_block_S_min']\n", "train_features_10['lelf_right_V_min'] = test_features['left_block_V_min'] - test_features['right_block_V_min']\n", "\n", "train_features_10['lelf_right_l_min'] = test_features['left_block_l_min'] - test_features['right_block_l_min']\n", "train_features_10['lelf_right_a_min'] = test_features['left_block_a_min'] - test_features['right_block_a_min']\n", "train_features_10['lelf_right_b_min'] = test_features['left_block_b_min'] - test_features['right_block_b_min']\n", "\n", "# train_features_10['left_grayValue']= test_features['left_grayValue'];\n", "# train_features_10['left_grayStddevValue']= test_features['left_grayStddevValue'];\n", "# train_features_10['left_grayHist']= test_features['left_grayHist'];\n", "# train_features_10['left_grayMax']= test_features['left_grayMax'];\n", "# train_features_10['left_grayMin']= test_features['left_grayMin'];\n", "\n", "# train_features_10['right_grayValue']= test_features['right_grayValue'];\n", "# train_features_10['right_grayStddevValue']= test_features['right_grayStddevValue'];\n", "# train_features_10['right_grayHist']= test_features['right_grayHist'];\n", "# train_features_10['right_grayMax']= test_features['right_grayMax'];\n", "# train_features_10['right_grayMin']= test_features['right_grayMin'];\n", "\n", "# train_features_10['lelf_R_stddev'] = test_features['left_block_R_stddev'] \n", "# train_features_10['lelf_G_stddev'] = test_features['left_block_G_stddev'] \n", "# train_features_10['lelf_B_stddev'] = test_features['left_block_B_stddev'] \n", "\n", "# train_features_10['left_block_R_min'] = test_features['left_block_R_min'] \n", "# train_features_10['left_block_G_min'] = test_features['left_block_G_min'] \n", "# train_features_10['left_block_B_min'] = test_features['left_block_B_min'] \n", "\n", "\n", "\n", "train_features_10['lelf_right_gray_value'] = test_features['left_grayValue'] - test_features['right_grayValue']\n", "train_features_10['lelf_right_gray_stddev'] = test_features['left_grayStddevValue'] - test_features['right_grayStddevValue']\n", "train_features_10['lelf_right_gray_hist'] = test_features['left_grayHist'] - test_features['right_grayHist']\n", "train_features_10['lelf_right_gray_max'] = test_features['left_grayMax'] - test_features['right_grayMax']\n", "train_features_10['lelf_right_gray_min'] = test_features['left_grayMin'] - test_features['right_grayMin']\n", "\n", "train_features_10.describe()\n", "\n", "\n", "# feature = feature.drop(\"left_block_H_hist\",axis=1)\n", "# feature = feature.drop(\"right_block_H_hist\",axis=1)\n", "# feature = feature.drop(\"whiteBlock_H_hist\",axis=1)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ " \n", "test_features = test_features.drop(\"left_block_H\",axis=1)\n", "test_features = test_features.drop(\"left_block_S\",axis=1)\n", "test_features = test_features.drop(\"left_block_V\",axis=1)\n", "\n", "test_features = test_features.drop(\"right_block_H\",axis=1)\n", "test_features = test_features.drop(\"right_block_S\",axis=1)\n", "test_features = test_features.drop(\"right_block_V\",axis=1)\n", "\n", "test_features = test_features.drop(\"whiteBlock_H\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_S\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_V\",axis=1)\n", "\n", "\n", "test_features = test_features.drop(\"left_block_H_stddev\",axis=1)\n", "test_features = test_features.drop(\"left_block_S_stddev\",axis=1)\n", "test_features = test_features.drop(\"left_block_V_stddev\",axis=1)\n", "\n", "test_features = test_features.drop(\"right_block_H_stddev\",axis=1)\n", "test_features = test_features.drop(\"right_block_S_stddev\",axis=1)\n", "test_features = test_features.drop(\"right_block_V_stddev\",axis=1)\n", "\n", "test_features = test_features.drop(\"whiteBlock_H_stddev\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_S_stddev\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_V_stddev\",axis=1)\n", "\n", "test_features = test_features.drop(\"left_block_H_hist\",axis=1)\n", "test_features = test_features.drop(\"left_block_S_hist\",axis=1)\n", "test_features = test_features.drop(\"left_block_V_hist\",axis=1)\n", "\n", "test_features = test_features.drop(\"right_block_H_hist\",axis=1)\n", "test_features = test_features.drop(\"right_block_S_hist\",axis=1)\n", "test_features = test_features.drop(\"right_block_V_hist\",axis=1)\n", "\n", "test_features = test_features.drop(\"whiteBlock_H_hist\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_S_hist\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_V_hist\",axis=1)\n", "\n", "test_features = test_features.drop(\"left_block_H_max\",axis=1)\n", "test_features = test_features.drop(\"left_block_S_max\",axis=1)\n", "test_features = test_features.drop(\"left_block_V_max\",axis=1)\n", "\n", "test_features = test_features.drop(\"right_block_H_max\",axis=1)\n", "test_features = test_features.drop(\"right_block_S_max\",axis=1)\n", "test_features = test_features.drop(\"right_block_V_max\",axis=1)\n", "\n", "test_features = test_features.drop(\"whiteBlock_H_max\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_S_max\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_V_max\",axis=1)\n", "\n", "test_features = test_features.drop(\"left_block_H_min\",axis=1)\n", "test_features = test_features.drop(\"left_block_S_min\",axis=1)\n", "test_features = test_features.drop(\"left_block_V_min\",axis=1)\n", "\n", "test_features = test_features.drop(\"right_block_H_min\",axis=1)\n", "test_features = test_features.drop(\"right_block_S_min\",axis=1)\n", "test_features = test_features.drop(\"right_block_V_min\",axis=1)\n", "\n", "test_features = test_features.drop(\"whiteBlock_H_min\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_S_min\",axis=1)\n", "test_features = test_features.drop(\"whiteBlock_V_min\",axis=1)\n", " \n", " \n", "test_features['lelf_right_R'] = test_features['left_block_R'] - test_features['right_block_R']\n", "test_features['lelf_right_G'] = test_features['left_block_G'] - test_features['right_block_G']\n", "test_features['lelf_right_B'] = test_features['left_block_B'] - test_features['right_block_B']\n", "\n", "# test_features['lelf_right_H'] = test_features['left_block_H'] - test_features['right_block_H']\n", "# test_features['lelf_right_S'] = test_features['left_block_S'] - test_features['right_block_S']\n", "# test_features['lelf_right_V'] = test_features['left_block_V'] - test_features['right_block_V']\n", "\n", "# test_features['lelf_right_l'] = test_features['left_block_l'] - test_features['right_block_l']\n", "# test_features['lelf_right_a'] = test_features['left_block_a'] - test_features['right_block_a']\n", "# test_features['lelf_right_b'] = test_features['left_block_b'] - test_features['right_block_b']\n", "\n", "# test_features['lelf_right_R_stddev'] = test_features['left_block_R_stddev'] - test_features['right_block_R_stddev']\n", "# test_features['lelf_right_G_stddev'] = test_features['left_block_G_stddev'] - test_features['right_block_G_stddev']\n", "# test_features['lelf_right_B_stddev'] = test_features['left_block_B_stddev'] - test_features['right_block_B_stddev']\n", "\n", "# test_features['lelf_right_H_stddev'] = test_features['left_block_H_stddev'] - test_features['right_block_H_stddev']\n", "# test_features['lelf_right_S_stddev'] = test_features['left_block_S_stddev'] - test_features['right_block_S_stddev']\n", "# test_features['lelf_right_V_stddev'] = test_features['left_block_V_stddev'] - test_features['right_block_V_stddev']\n", "\n", "# test_features['lelf_right_l_stddev'] = test_features['left_block_l_stddev'] - test_features['right_block_l_stddev']\n", "# test_features['lelf_right_a_stddev'] = test_features['left_block_a_stddev'] - test_features['right_block_a_stddev']\n", "# test_features['lelf_right_b_stddev'] = test_features['left_block_b_stddev'] - test_features['right_block_b_stddev']\n", "\n", "# test_features['lelf_right_R_hist'] = test_features['left_block_R_hist'] - test_features['right_block_R_hist']\n", "# test_features['lelf_right_G_hist'] = test_features['left_block_G_hist'] - test_features['right_block_G_hist']\n", "# test_features['lelf_right_B_hist'] = test_features['left_block_B_hist'] - test_features['right_block_B_hist']\n", "\n", "# test_features['lelf_right_H_hist'] = test_features['left_block_H_hist'] - test_features['right_block_H_hist']\n", "# test_features['lelf_right_S_hist'] = test_features['left_block_S_hist'] - test_features['right_block_S_hist']\n", "# test_features['lelf_right_V_hist'] = test_features['left_block_V_hist'] - test_features['right_block_V_hist']\n", "\n", "# test_features['lelf_right_l_hist'] = test_features['left_block_l_hist'] - test_features['right_block_l_hist']\n", "# test_features['lelf_right_a_hist'] = test_features['left_block_a_hist'] - test_features['right_block_a_hist']\n", "# test_features['lelf_right_b_hist'] = test_features['left_block_b_hist'] - test_features['right_block_b_hist']\n", "\n", "# test_features['lelf_right_R_max'] = test_features['left_block_R_max'] - test_features['right_block_R_max']\n", "# test_features['lelf_right_G_max'] = test_features['left_block_G_max'] - test_features['right_block_G_max']\n", "# test_features['lelf_right_B_max'] = test_features['left_block_B_max'] - test_features['right_block_B_max']\n", "\n", "# test_features['lelf_right_H_max'] = test_features['left_block_H_max'] - test_features['right_block_H_max']\n", "# test_features['lelf_right_S_max'] = test_features['left_block_S_max'] - test_features['right_block_S_max']\n", "# test_features['lelf_right_V_max'] = test_features['left_block_V_max'] - test_features['right_block_V_max']\n", "\n", "# test_features['lelf_right_l_max'] = test_features['left_block_l_max'] - test_features['right_block_l_max']\n", "# test_features['lelf_right_a_max'] = test_features['left_block_a_max'] - test_features['right_block_a_max']\n", "# test_features['lelf_right_b_max'] = test_features['left_block_b_max'] - test_features['right_block_b_max']\n", "\n", "\n", "\n", "# test_features['lelf_right_R_min'] = test_features['left_block_R_min'] - test_features['right_block_R_min']\n", "# test_features['lelf_right_G_min'] = test_features['left_block_G_min'] - test_features['right_block_G_min']\n", "# test_features['lelf_right_B_min'] = test_features['left_block_B_min'] - test_features['right_block_B_min']\n", "\n", "# test_features['lelf_right_H_min'] = test_features['left_block_H_min'] - test_features['right_block_H_min']\n", "# test_features['lelf_right_S_min'] = test_features['left_block_S_min'] - test_features['right_block_S_min']\n", "# test_features['lelf_right_V_min'] = test_features['left_block_V_min'] - test_features['right_block_V_min']\n", "\n", "# test_features['lelf_right_l_min'] = test_features['left_block_l_min'] - test_features['right_block_l_min']\n", "# test_features['lelf_right_a_min'] = test_features['left_block_a_min'] - test_features['right_block_a_min']\n", "# test_features['lelf_right_b_min'] = test_features['left_block_b_min'] - test_features['right_block_b_min']\n", "\n", "test_features['lelf_right_gray_value'] = test_features['left_grayValue'] - test_features['right_grayValue']\n", "test_features['lelf_right_gray_stddev'] = test_features['left_grayStddevValue'] - test_features['right_grayStddevValue']\n", "test_features['lelf_right_gray_hist'] = test_features['left_grayHist'] - test_features['right_grayHist']\n", "test_features['lelf_right_gray_max'] = test_features['left_grayMax'] - test_features['right_grayMax']\n", "test_features['lelf_right_gray_min'] = test_features['left_grayMin'] - test_features['right_grayMin']\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pred = clf_svm_linear.predict(train_features_10)\n", "test_features_gray_stddev = test_features['left_grayStddevValue']\n", "test_features_np = np.ndarray(test_features_gray_stddev.shape,dtype = np.float32)\n", "\n", "test_features_np = test_features_gray_stddev.values\n", "print \"svm linear accuracy score:\" , accuracy_score(test_labels,pred)\n", "print \"f1 score:\" , f1_score(test_labels,pred,average='micro')\n", "print \"recall_score :\" , recall_score(test_labels,pred,average='micro')\n", "print \"precision_score :\" , precision_score(test_labels,pred,average='micro')\n", "\n", "for i in range(0, len(test_features_np)):\n", " if test_features_np[i] < 3:\n", " pred[i] =0\n", "print \"svm linear accuracy score:\" , accuracy_score(test_labels,pred)\n", "print \"f1 score:\" , f1_score(test_labels,pred,average='micro')\n", "print \"recall_score :\" , recall_score(test_labels,pred,average='micro')\n", "print \"precision_score :\" , precision_score(test_labels,pred,average='micro')\n", "\n", "\n", "print(\"preds:\",pred[120:130])\n", "print('trues:\\n',test_labels[120:130])\n", "test_labels_np = np.ndarray(test_labels.shape,dtype= np.int32)\n", "test_labels_np = test_labels.values\n", "print(test_labels_np[0])\n", "all_counter = 0\n", "counter = 0\n", "for i in range(0 ,len(pred) ):\n", " if (pred[i] == 4 or (pred[i] == 4 and test_labels_np[i] ==4 )or test_labels_np[i] ==4 ) :\n", " all_counter = all_counter + 1\n", " if pred[i] != test_labels_np[i] :\n", " counter = counter+1\n", " print(pred[i] , test_labels_np[i])\n", "print(len(pred),all_counter, counter) \n", "all_counter = 0\n", "counter = 0\n", "for i in range(0 ,len(pred) ):\n", " if pred[i] != test_labels_np[i] :\n", " counter = counter+1\n", " print(pred[i] , test_labels_np[i])\n", "print(len(pred),all_counter, counter) \n", "\n", "# print \"svm linear accuracy score:\" , accuracy_score(test_labels,pred)\n", "# print \"f1 score:\" , f1_score(test_labels,pred,average='micro')\n", "# print \"recall_score :\" , recall_score(test_labels,pred,average='micro')\n", "# print \"precision_score :\" , precision_score(test_labels,pred,average='micro')\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## from sklearn.metrics import recall_score\n", "from sklearn.metrics import precision_score\n", "print \"accuracy score:\" , accuracy_score(y_test,pred)\n", "print \"recall_score :\" , recall_score(y_test,pred,average='macro')\n", "print \"precision_score :\" , precision_score(y_test,pred,average='macro')\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn_porter import Porter\n", "\n", "porter_java = Porter(clf_svm, language='java').export()\n", "porter_c = Porter(clf_svm, language='c').export()\n", "\n", "f = open(\"Protein_c.txt\",'wb')\n", "f.write(porter_c)\n", "f.close()\n", "\n", "f = open(\"Protein_svm_java.txt\",'wb')\n", "f.write(porter_java)\n", "f.close()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "try :\n", " data = pd.read_csv(\"data.csv\")\n", " print (\"load data successful !!!!!\")\n", " data0 = data[data[\"index\"] == 0]\n", " data1 = data[data[\"index\"] == 1]\n", " data2 = data[data[\"index\"] == 2]\n", " data3 = data[data[\"index\"] == 3]\n", " data4 = data[data[\"index\"] == 4]\n", " data0.to_csv('data0.csv')\n", " data1.to_csv('data1.csv')\n", " data2.to_csv('data2.csv')\n", " data3.to_csv('data3.csv')\n", " data4.to_csv('data4.csv') \n", "except :\n", " print (\"load data error !!!!!!!!!!\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.4" } }, "nbformat": 4, "nbformat_minor": 2 }