when_dataset_is_small_features_are_your_friend.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0573d192-f2f4-4916-9120-79c76e08483c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "from torch import nn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from skorch import NeuralNetClassifier\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau, CyclicLR\n",
    "from skorch.callbacks import LRScheduler\n",
    "from sklearn.metrics import accuracy_score\n",
    "from scipy.fft import fft\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7f2d7f4e-5ba8-4b34-b32f-edd7d41fc211",
   "metadata": {},
   "outputs": [],
   "source": [
    "def signal0(samples_per_signal, noise_amplitude):\n",
    "    x = np.linspace(0, 4.0, samples_per_signal)\n",
    "    y = np.sin(x * np.pi * 0.5)\n",
    "    n = np.random.randn(samples_per_signal) * noise_amplitude\n",
    "    \n",
    "    s = y + n\n",
    "    \n",
    "    shift = np.random.randint(low=0, high=int(samples_per_signal / 2))\n",
    "    s = np.concatenate([s[shift:], s[:shift]])\n",
    "    \n",
    "    return np.asarray(s, dtype=np.float32)\n",
    "    \n",
    "\n",
    "def signal1(samples_per_signal, noise_amplitude):\n",
    "    x = np.linspace(0, 4.0, samples_per_signal)\n",
    "    y = np.sin(x * np.pi)\n",
    "    n = np.random.randn(samples_per_signal) * noise_amplitude\n",
    "    \n",
    "    s = y + n\n",
    "    \n",
    "    shift = np.random.randint(low=0, high=int(samples_per_signal / 2))\n",
    "    s = np.concatenate([s[shift:], s[:shift]])\n",
    "    \n",
    "    return np.asarray(s, dtype=np.float32)\n",
    "\n",
    "\n",
    "\n",
    "class Network(nn.Module):\n",
    "    \n",
    "    def __init__(self, signal_size):\n",
    "        \n",
    "        c = int(signal_size / 10)\n",
    "        if c < 3:\n",
    "            c = 3\n",
    "        \n",
    "        super().__init__()\n",
    "        self.cnn = nn.Sequential(\n",
    "            nn.Conv1d(1, 8, c),\n",
    "            nn.ReLU(),\n",
    "            #nn.Dropout1d(0.1),\n",
    "            nn.AvgPool1d(2),\n",
    "            nn.Conv1d(8, 16, c),\n",
    "            nn.ReLU(),\n",
    "            #nn.Dropout1d(0.1),\n",
    "            nn.AvgPool1d(2),\n",
    "            #nn.Conv1d(16, 1, 1),\n",
    "            #nn.Dropout1d(0.1),\n",
    "            nn.ReLU(),\n",
    "            nn.Flatten()\n",
    "        )\n",
    "        \n",
    "        l = 0\n",
    "        with torch.no_grad():\n",
    "            s = torch.randn((1,1,SAMPLES_PER_SIGNAL))\n",
    "            o = self.cnn(s)\n",
    "            l = o.shape[1]\n",
    "        \n",
    "        self.head = nn.Sequential(\n",
    "            nn.Linear(l, 2 * l),\n",
    "            nn.ReLU(),\n",
    "            #nn.Dropout1d(0.1),\n",
    "            nn.Linear(2 * l, 2),\n",
    "            nn.ReLU(),\n",
    "            nn.Softmax(dim=1)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = self.cnn(x)\n",
    "        x = self.head(x)\n",
    "        \n",
    "        return x    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "da66e39d-c03c-49e7-b795-fad3a1992376",
   "metadata": {},
   "outputs": [],
   "source": [
    "SAMPLES_PER_SIGNAL = 100\n",
    "SIGNALS_IN_DATASET = 20\n",
    "NOISE_AMPLITUDE = 0.1\n",
    "REPEAT_EXPERIMENT = 10\n",
    "\n",
    "X, Y = [], []\n",
    "\n",
    "stop = int(SIGNALS_IN_DATASET / 2)\n",
    "for i in range(SIGNALS_IN_DATASET):\n",
    "\n",
    "    if i < stop:\n",
    "        x = signal0(SAMPLES_PER_SIGNAL, NOISE_AMPLITUDE)\n",
    "        y = 0\n",
    "    else:\n",
    "        x = signal1(SAMPLES_PER_SIGNAL, NOISE_AMPLITUDE)\n",
    "        y = 1\n",
    "\n",
    "    X.append(x.reshape(1,-1))\n",
    "    Y.append(y)\n",
    "\n",
    "X = np.concatenate(X)\n",
    "Y = np.array(Y, dtype=np.int64)\n",
    "\n",
    "train_x, test_x, train_y, test_y = train_test_split(X, Y, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8bb75b72-38b5-4a98-b63c-398fd1618b82",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 - 0.55\n",
      "Train Acc: 0.49444444444444446, Test Acc: 0.55\n",
      "1 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "2 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "3 - 0.55\n",
      "Train Acc: 0.49444444444444446, Test Acc: 0.55\n",
      "4 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "5 - 0.55\n",
      "Train Acc: 0.49444444444444446, Test Acc: 0.55\n",
      "6 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "7 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "8 - 0.95\n",
      "Train Acc: 0.9722222222222222, Test Acc: 0.95\n",
      "9 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "Average acc: 0.8600000000000001\n",
      "Average train acc: 0.8455555555555556\n",
      "Average acc where training was successful: 0.9928571428571429\n",
      "Training success rate: 0.7\n"
     ]
    }
   ],
   "source": [
    "accs = []\n",
    "train_accs = []\n",
    "\n",
    "for i in range(REPEAT_EXPERIMENT):\n",
    "\n",
    "    net = NeuralNetClassifier(\n",
    "        lambda: Network(SAMPLES_PER_SIGNAL),\n",
    "        max_epochs=200,\n",
    "        criterion=nn.CrossEntropyLoss(),\n",
    "        lr=0.1,\n",
    "        callbacks=[\n",
    "            #('lr_scheduler', LRScheduler(policy=ReduceLROnPlateau, monitor=\"valid_acc\", mode=\"min\", verbose=True)),\n",
    "            ('lr_scheduler', LRScheduler(policy=CyclicLR, base_lr=0.0001, max_lr=0.01, step_size_up=10)),\n",
    "        ],\n",
    "        verbose=False,\n",
    "        batch_size=128\n",
    "    )\n",
    "\n",
    "    net = net.fit(train_x.reshape(train_x.shape[0], 1, SAMPLES_PER_SIGNAL), train_y)\n",
    "    pred = net.predict(test_x.reshape(test_x.shape[0], 1, SAMPLES_PER_SIGNAL))\n",
    "    acc = accuracy_score(test_y, pred)\n",
    "    \n",
    "    print(f\"{i} - {acc}\")\n",
    "\n",
    "\n",
    "    accs.append(acc)\n",
    "    \n",
    "    pred_train = net.predict(train_x.reshape(train_x.shape[0], 1, SAMPLES_PER_SIGNAL))\n",
    "    train_acc = accuracy_score(train_y, pred_train)\n",
    "    train_accs.append(train_acc)\n",
    "    \n",
    "    print(f\"Train Acc: {train_acc}, Test Acc: {acc}\")\n",
    "    \n",
    "accs = np.array(accs)\n",
    "train_accs = np.array(train_accs)\n",
    "\n",
    "print(f\"Average acc: {accs.mean()}\")\n",
    "print(f\"Average train acc: {train_accs.mean()}\")\n",
    "print(f\"Average acc where training was successful: {accs[train_accs > 0.6].mean()}\")\n",
    "print(f\"Training success rate: {(train_accs > 0.6).mean()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "38a029f7-894f-457f-80d4-461f4d2e3724",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "1 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "2 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "3 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "4 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "5 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "6 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "7 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "8 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "9 - 1.0\n",
      "Train Acc: 1.0, Test Acc: 1.0\n",
      "Average acc: 1.0\n",
      "Average train acc: 1.0\n",
      "Average acc where training was successful: 1.0\n",
      "Training success rate: 1.0\n"
     ]
    }
   ],
   "source": [
    "X, Y = [], []\n",
    "\n",
    "stop = int(SIGNALS_IN_DATASET / 2)\n",
    "for i in range(SIGNALS_IN_DATASET):\n",
    "\n",
    "    if i < stop:\n",
    "        x = signal0(SAMPLES_PER_SIGNAL, NOISE_AMPLITUDE)\n",
    "        y = 0\n",
    "    else:\n",
    "        x = signal1(SAMPLES_PER_SIGNAL, NOISE_AMPLITUDE)\n",
    "        y = 1\n",
    "\n",
    "    x = np.abs(fft(x[:int(SAMPLES_PER_SIGNAL /2 )]))    \n",
    "    \n",
    "    X.append(x.reshape(1,-1))\n",
    "    Y.append(y)\n",
    "\n",
    "X = np.concatenate(X)\n",
    "Y = np.array(Y, dtype=np.int64)\n",
    "\n",
    "train_x, test_x, train_y, test_y = train_test_split(X, Y, test_size=0.1)\n",
    "\n",
    "accs = []\n",
    "train_accs = []\n",
    "\n",
    "for i in range(REPEAT_EXPERIMENT):\n",
    "    model = RandomForestClassifier()\n",
    "    model.fit(train_x, train_y)\n",
    "    \n",
    "    pred = model.predict(test_x)\n",
    "    acc = accuracy_score(test_y, pred)\n",
    "    \n",
    "    print(f\"{i} - {acc}\")\n",
    "\n",
    "\n",
    "    accs.append(acc)\n",
    "    \n",
    "    pred_train = model.predict(train_x)\n",
    "    train_acc = accuracy_score(train_y, pred_train)\n",
    "    train_accs.append(train_acc)\n",
    "    \n",
    "    print(f\"Train Acc: {train_acc}, Test Acc: {acc}\")\n",
    "    \n",
    "accs = np.array(accs)\n",
    "train_accs = np.array(train_accs)\n",
    "\n",
    "print(f\"Average acc: {accs.mean()}\")\n",
    "print(f\"Average train acc: {train_accs.mean()}\")\n",
    "print(f\"Average acc where training was successful: {accs[train_accs > 0.6].mean()}\")\n",
    "print(f\"Training success rate: {(train_accs > 0.6).mean()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c1da3e7-54c3-4b08-9793-7a4d3a909227",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}