-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix #1 by adding ensemble two moons notebook
- Loading branch information
y0ast
committed
Sep 21, 2020
1 parent
2adbca7
commit 93321fa
Showing
1 changed file
with
194 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,194 @@ | ||
{ | ||
"cells": [ | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"import torch\n", | ||
"import torch.utils.data\n", | ||
"from torch import nn\n", | ||
"from torch.nn import functional as F\n", | ||
"\n", | ||
"from ignite.engine import Events, Engine\n", | ||
"from ignite.metrics import Accuracy, Loss\n", | ||
"\n", | ||
"import numpy as np\n", | ||
"import sklearn.datasets\n", | ||
"\n", | ||
"import matplotlib.pyplot as plt\n", | ||
"import seaborn as sns\n", | ||
"\n", | ||
"sns.set()\n", | ||
"\n", | ||
"torch.manual_seed(1)\n", | ||
"np.random.seed(1)" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"class Model(nn.Module):\n", | ||
" def __init__(self, features):\n", | ||
" super().__init__()\n", | ||
" \n", | ||
" self.fc1 = nn.Linear(2, features)\n", | ||
" self.fc2 = nn.Linear(features, features)\n", | ||
" self.fc3 = nn.Linear(features, features)\n", | ||
" self.fc4 = nn.Linear(features, 2)\n", | ||
"\n", | ||
" def forward(self, x):\n", | ||
" x = F.relu(self.fc1(x))\n", | ||
" x = F.relu(self.fc2(x))\n", | ||
" x = F.relu(self.fc3(x))\n", | ||
" x = self.fc4(x)\n", | ||
" \n", | ||
" return F.log_softmax(x, dim=1)" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"noise = 0.1\n", | ||
"\n", | ||
"X_train, y_train = sklearn.datasets.make_moons(n_samples=1000, noise=noise)\n", | ||
"X_test, y_test = sklearn.datasets.make_moons(n_samples=200, noise=noise)\n", | ||
"\n", | ||
"num_classes = 2\n", | ||
"batch_size = 64\n", | ||
"\n", | ||
"def train_model(max_epochs):\n", | ||
" model = Model(20)\n", | ||
"\n", | ||
" optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4)\n", | ||
"\n", | ||
" def step(engine, batch):\n", | ||
" model.train()\n", | ||
" optimizer.zero_grad()\n", | ||
"\n", | ||
" x, y = batch\n", | ||
"\n", | ||
" y_pred = model(x)\n", | ||
" loss = F.nll_loss(y_pred, y)\n", | ||
"\n", | ||
" loss.backward()\n", | ||
" optimizer.step()\n", | ||
"\n", | ||
" return loss.item()\n", | ||
"\n", | ||
" def eval_step(engine, batch):\n", | ||
" model.eval()\n", | ||
"\n", | ||
" x, y = batch\n", | ||
" y_pred = model(x)\n", | ||
"\n", | ||
" return y_pred, y\n", | ||
"\n", | ||
"\n", | ||
" trainer = Engine(step)\n", | ||
" evaluator = Engine(eval_step)\n", | ||
"\n", | ||
" metric = Accuracy()\n", | ||
" metric.attach(evaluator, \"accuracy\")\n", | ||
"\n", | ||
" metric = Loss(F.nll_loss)\n", | ||
" metric.attach(evaluator, \"nll\")\n", | ||
"\n", | ||
" ds_train = torch.utils.data.TensorDataset(torch.from_numpy(X_train).float(), torch.from_numpy(y_train))\n", | ||
" dl_train = torch.utils.data.DataLoader(ds_train, batch_size=batch_size, shuffle=True, drop_last=True)\n", | ||
"\n", | ||
" ds_test = torch.utils.data.TensorDataset(torch.from_numpy(X_test).float(), torch.from_numpy(y_test))\n", | ||
" dl_test = torch.utils.data.DataLoader(ds_test, batch_size=200, shuffle=False)\n", | ||
"\n", | ||
" @trainer.on(Events.EPOCH_COMPLETED)\n", | ||
" def log_results(trainer):\n", | ||
" evaluator.run(dl_test)\n", | ||
" metrics = evaluator.state.metrics\n", | ||
"\n", | ||
" print(f\"Test Results - Epoch: {trainer.state.epoch} Acc: {metrics['accuracy']:.4f} NLL: {metrics['nll']:.2f}\")\n", | ||
" \n", | ||
" trainer.run(dl_train, max_epochs=max_epochs)\n", | ||
" \n", | ||
" return model" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"metadata": { | ||
"scrolled": false | ||
}, | ||
"outputs": [], | ||
"source": [ | ||
"ensemble = 5\n", | ||
"models = [train_model(50) for _ in range(ensemble)]" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"domain = 3\n", | ||
"x = np.linspace(-domain+0.5, domain+0.5, 100)\n", | ||
"y = np.linspace(-domain, domain, 100)\n", | ||
"\n", | ||
"xx, yy = np.meshgrid(x, y)\n", | ||
"\n", | ||
"X = np.column_stack([xx.flatten(), yy.flatten()])\n", | ||
"\n", | ||
"X_vis, y_vis = sklearn.datasets.make_moons(n_samples=500, noise=noise)\n", | ||
"mask = y_vis.astype(np.bool)\n", | ||
"\n", | ||
"for model in models:\n", | ||
" model.eval()\n", | ||
"\n", | ||
"with torch.no_grad():\n", | ||
" predictions = torch.stack([model(torch.from_numpy(X).float()) for model in models])\n", | ||
"\n", | ||
" mean_prediction = torch.mean(predictions.exp(), dim=0)\n", | ||
" confidence = torch.sum(mean_prediction * torch.log(mean_prediction), dim=1)\n", | ||
"\n", | ||
"z = confidence.reshape(xx.shape)\n", | ||
"\n", | ||
"plt.figure()\n", | ||
"plt.contourf(x, y, z, cmap='cividis')\n", | ||
"\n", | ||
"plt.scatter(X_vis[mask,0], X_vis[mask,1])\n", | ||
"plt.scatter(X_vis[~mask,0], X_vis[~mask,1])\n", | ||
"\n", | ||
"plt.figure()\n", | ||
"plt.contourf(x, y, z, cmap='cividis')" | ||
] | ||
} | ||
], | ||
"metadata": { | ||
"kernelspec": { | ||
"display_name": "Python 3", | ||
"language": "python", | ||
"name": "python3" | ||
}, | ||
"language_info": { | ||
"codemirror_mode": { | ||
"name": "ipython", | ||
"version": 3 | ||
}, | ||
"file_extension": ".py", | ||
"mimetype": "text/x-python", | ||
"name": "python", | ||
"nbconvert_exporter": "python", | ||
"pygments_lexer": "ipython3", | ||
"version": "3.8.5" | ||
} | ||
}, | ||
"nbformat": 4, | ||
"nbformat_minor": 2 | ||
} |