diff --git a/00_core.ipynb b/00_core.ipynb deleted file mode 100644 index 6c17de9..0000000 --- a/00_core.ipynb +++ /dev/null @@ -1,48 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp core" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# module name here\n", - "\n", - "> API details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#hide\n", - "from nbdev.showdoc import *" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/Nbs/00_ModelConstructor.ipynb b/Nbs/00_ModelConstructor.ipynb new file mode 100644 index 0000000..ecf9ee5 --- /dev/null +++ b/Nbs/00_ModelConstructor.ipynb @@ -0,0 +1,1597 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Model constructor.\n", + "\n", + "> Create and tune pytorch model." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "#hide" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:11:43.985194Z", + "start_time": "2021-10-11T11:11:43.178837Z" + } + }, + "outputs": [], + "source": [ + "#hide\n", + "import torch\n", + "import torch.nn as nn\n", + "\n", + "from nbdev.showdoc import show_doc\n", + "from IPython.display import Markdown, display" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:11:46.969707Z", + "start_time": "2021-10-11T11:11:46.964854Z" + } + }, + "outputs": [], + "source": [ + "# hide\n", + "def print_doc(func_name):\n", + " doc = show_doc(func_name, title_level=4, disp=False)\n", + " display(Markdown(doc))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ResBlock" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:12:51.179027Z", + "start_time": "2021-10-11T11:12:51.175828Z" + } + }, + "outputs": [], + "source": [ + "#hide\n", + "from model_constructor.model_constructor import ResBlock" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:13:16.700317Z", + "start_time": "2021-10-11T11:13:16.692963Z" + } + }, + "outputs": [ + { + "data": { + "text/markdown": [ + "

class ResBlock[source]

\n", + "\n", + "> ResBlock(**`expansion`**, **`ni`**, **`nh`**, **`stride`**=*`1`*, **`conv_layer`**=*`ConvLayer`*, **`act_fn`**=*`ReLU(inplace=True)`*, **`zero_bn`**=*`True`*, **`bn_1st`**=*`True`*, **`pool`**=*`AvgPool2d(kernel_size=2, stride=2, padding=0)`*, **`sa`**=*`False`*, **`sym`**=*`False`*, **`groups`**=*`1`*, **`dw`**=*`False`*, **`div_groups`**=*`None`*, **`se_module`**=*`SEModule`*, **`se`**=*`False`*, **`se_reduction`**=*`16`*) :: `Module`\n", + "\n", + "Resnet block" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#hide_input\n", + "print_doc(ResBlock)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:19:42.319165Z", + "start_time": "2021-10-11T11:19:42.313079Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (sa): SimpleSelfAttention(\n", + " (conv): Conv1d(64, 64, kernel_size=(1,), stride=(1,), bias=False)\n", + " )\n", + " )\n", + " (act_fn): ReLU(inplace=True)\n", + ")" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "block = ResBlock(1,64,64,sa=True)\n", + "block" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:20:46.755909Z", + "start_time": "2021-10-11T11:20:46.705654Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 64, 32, 32])\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = block(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:21:16.759188Z", + "start_time": "2021-10-11T11:21:16.750479Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64, bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (sa): SimpleSelfAttention(\n", + " (conv): Conv1d(256, 256, kernel_size=(1,), stride=(1,), bias=False)\n", + " )\n", + " )\n", + " (act_fn): ReLU(inplace=True)\n", + ")" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "block = ResBlock(4,64,64,sa=True, dw=True)\n", + "block" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:22:39.905684Z", + "start_time": "2021-10-11T11:22:39.769037Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 256, 32, 32])\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 256, 32, 32)\n", + "y = block(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 256, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:22:40.575583Z", + "start_time": "2021-10-11T11:22:40.567684Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=4, bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (sa): SimpleSelfAttention(\n", + " (conv): Conv1d(256, 256, kernel_size=(1,), stride=(1,), bias=False)\n", + " )\n", + " )\n", + " (act_fn): ReLU(inplace=True)\n", + ")" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "block = ResBlock(4,64,64,sa=True, groups=4)\n", + "block" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:22:41.502348Z", + "start_time": "2021-10-11T11:22:41.376582Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 256, 32, 32])\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 256, 32, 32)\n", + "y = block(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 256, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:23:02.825620Z", + "start_time": "2021-10-11T11:23:02.818437Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " )\n", + " (act_fn): LeakyReLU(negative_slope=0.01)\n", + ")" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "block = ResBlock(2,64,64,act_fn=nn.LeakyReLU(), bn_1st=False)\n", + "block" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:23:35.186158Z", + "start_time": "2021-10-11T11:23:35.139894Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 128, 32, 32])\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 128, 32, 32)\n", + "y = block(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 128, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:23:45.637724Z", + "start_time": "2021-10-11T11:23:45.630146Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (se): SEModule(\n", + " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", + " (excitation): Sequential(\n", + " (fc_reduce): Linear(in_features=128, out_features=8, bias=True)\n", + " (se_act): ReLU(inplace=True)\n", + " (fc_expand): Linear(in_features=8, out_features=128, bias=True)\n", + " (se_gate): Sigmoid()\n", + " )\n", + " )\n", + " (sa): SimpleSelfAttention(\n", + " (conv): Conv1d(128, 128, kernel_size=(1,), stride=(1,), bias=False)\n", + " )\n", + " )\n", + " (act_fn): ReLU(inplace=True)\n", + ")" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "block = ResBlock(2, 64, 64, sa=True, se=True)\n", + "block" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:23:57.314724Z", + "start_time": "2021-10-11T11:23:57.243225Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 128, 32, 32])\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 128, 32, 32)\n", + "y = block(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 128, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Stem, Body, Head" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:15:59.922762Z", + "start_time": "2021-10-11T11:15:59.919126Z" + } + }, + "outputs": [], + "source": [ + "#hide\n", + "from model_constructor.model_constructor import _make_body, _make_head, _make_layer, _make_stem" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:16:13.371233Z", + "start_time": "2021-10-11T11:16:13.366267Z" + } + }, + "outputs": [ + { + "data": { + "text/markdown": [ + "

_make_layer[source]

\n", + "\n", + "> _make_layer(**`expansion`**, **`ni`**, **`nf`**, **`blocks`**, **`stride`**, **`sa`**)\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#hide_input\n", + "print_doc(_make_layer)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:17:17.297858Z", + "start_time": "2021-10-11T11:17:17.291866Z" + } + }, + "outputs": [ + { + "data": { + "text/markdown": [ + "

_make_stem[source]

\n", + "\n", + "> _make_stem()\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#hide_input\n", + "print_doc(_make_stem)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:17:17.297858Z", + "start_time": "2021-10-11T11:17:17.291866Z" + } + }, + "outputs": [ + { + "data": { + "text/markdown": [ + "

_make_body[source]

\n", + "\n", + "> _make_body()\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#hide_input\n", + "print_doc(_make_body)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:17:17.297858Z", + "start_time": "2021-10-11T11:17:17.291866Z" + } + }, + "outputs": [ + { + "data": { + "text/markdown": [ + "

_make_head[source]

\n", + "\n", + "> _make_head()\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#hide_input\n", + "print_doc(_make_head)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model Constructor." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:18:41.974578Z", + "start_time": "2021-10-11T11:18:41.971385Z" + } + }, + "outputs": [], + "source": [ + "#hide\n", + "from model_constructor import ModelConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:18:59.147639Z", + "start_time": "2021-10-11T11:18:59.138898Z" + } + }, + "outputs": [ + { + "data": { + "text/markdown": [ + "

class ModelConstructor[source]

\n", + "\n", + "> ModelConstructor(**`name`**=*`'MC'`*, **`c_in`**=*`3`*, **`c_out`**=*`1000`*, **`block`**=*`ResBlock`*, **`conv_layer`**=*`ConvLayer`*, **`block_sizes`**=*`[64, 128, 256, 512]`*, **`layers`**=*`[2, 2, 2, 2]`*, **`norm`**=*`BatchNorm2d`*, **`act_fn`**=*`ReLU(inplace=True)`*, **`pool`**=*`AvgPool2d(kernel_size=2, stride=2, padding=0)`*, **`expansion`**=*`1`*, **`groups`**=*`1`*, **`dw`**=*`False`*, **`div_groups`**=*`None`*, **`sa`**=*`False`*, **`se`**=*`False`*, **`se_module`**=*`SEModule`*, **`se_reduction`**=*`16`*, **`bn_1st`**=*`True`*, **`zero_bn`**=*`True`*, **`stem_stride_on`**=*`0`*, **`stem_sizes`**=*`[32, 32, 64]`*, **`stem_pool`**=*`MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)`*, **`stem_bn_end`**=*`False`*, **`_init_cnn`**=*`init_cnn`*, **`_make_stem`**=*`_make_stem`*, **`_make_layer`**=*`_make_layer`*, **`_make_body`**=*`_make_body`*, **`_make_head`**=*`_make_head`*)\n", + "\n", + "Model constructor. As default - xresnet18" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#hide_input\n", + "print_doc(ModelConstructor)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:24:22.231480Z", + "start_time": "2021-10-11T11:24:22.226164Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "MC constructor\n", + " c_in: 3, c_out: 1000\n", + " expansion: 1, groups: 1, dw: False, div_groups: None\n", + " sa: False, se: False\n", + " stem sizes: [3, 32, 32, 64], stride on 0\n", + " body sizes [64, 128, 256, 512]\n", + " layers: [2, 2, 2, 2]" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mc = ModelConstructor()\n", + "mc" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:24:27.424129Z", + "start_time": "2021-10-11T11:24:27.419772Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[64, 128, 256, 512]" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mc._block_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:24:33.782053Z", + "start_time": "2021-10-11T11:24:33.777945Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[64, 64, 128, 256, 512]" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mc.block_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:24:45.776152Z", + "start_time": "2021-10-11T11:24:45.770558Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "MC constructor\n", + " c_in: 3, c_out: 1000\n", + " expansion: 1, groups: 1, dw: False, div_groups: None\n", + " sa: False, se: False\n", + " stem sizes: [3, 32, 32, 64], stride on 0\n", + " body sizes [128, 256, 512, 1024]\n", + " layers: [2, 2, 2, 2]" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mc._block_sizes = [128, 256, 512, 1024]\n", + "mc" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:24:52.391894Z", + "start_time": "2021-10-11T11:24:52.386609Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[64, 128, 256, 512, 1024]" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mc.block_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:25:09.676012Z", + "start_time": "2021-10-11T11:25:09.670424Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "MC constructor\n", + " c_in: 3, c_out: 1000\n", + " expansion: 1, groups: 1, dw: False, div_groups: None\n", + " sa: False, se: False\n", + " stem sizes: [3, 32, 32, 64], stride on 0\n", + " body sizes [64, 128, 256, 512]\n", + " layers: [2, 2, 2, 2]" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#hide\n", + "mc = ModelConstructor(stem_sizes=[3,32,32,64])\n", + "mc" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:25:16.570856Z", + "start_time": "2021-10-11T11:25:16.565501Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[64, 64, 128, 256, 512]" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mc.block_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:25:24.136712Z", + "start_time": "2021-10-11T11:25:24.132577Z" + } + }, + "outputs": [], + "source": [ + "model = ModelConstructor()" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "#hide\n", + "# model.block_sizes = [64, 128, 256, 512] # wrong way --> use _block_sizes\n", + "# model" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:25:38.451437Z", + "start_time": "2021-10-11T11:25:38.445228Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (stem_pool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n", + ")" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#collapse_output\n", + "mc.stem" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:25:45.355684Z", + "start_time": "2021-10-11T11:25:45.349844Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (act_fn): ReLU(inplace=True)\n", + " )\n", + " (stem_pool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n", + ")" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#collapse_output\n", + "mc.stem_stride_on = 1\n", + "mc.stem" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:25:57.582843Z", + "start_time": "2021-10-11T11:25:57.446266Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 64, 32, 32])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/jzz/anaconda3/envs/pt_19/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1623448278899/work/c10/core/TensorImpl.h:1156.)\n", + " return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 3, 128, 128)\n", + "y = mc.stem(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:26:05.321104Z", + "start_time": "2021-10-11T11:26:05.317555Z" + } + }, + "outputs": [], + "source": [ + "mc.bn_1st = False" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:26:11.411747Z", + "start_time": "2021-10-11T11:26:11.405690Z" + } + }, + "outputs": [], + "source": [ + "mc.act_fn =nn.LeakyReLU(inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:26:17.369743Z", + "start_time": "2021-10-11T11:26:17.365499Z" + } + }, + "outputs": [], + "source": [ + "mc.sa = True\n", + "mc.se = True" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:26:23.133910Z", + "start_time": "2021-10-11T11:26:23.071346Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential(\n", + " (bl_0): ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (se): SEModule(\n", + " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", + " (excitation): Sequential(\n", + " (fc_reduce): Linear(in_features=128, out_features=8, bias=True)\n", + " (se_act): ReLU(inplace=True)\n", + " (fc_expand): Linear(in_features=8, out_features=128, bias=True)\n", + " (se_gate): Sigmoid()\n", + " )\n", + " )\n", + " )\n", + " (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", + " (idconv): ConvLayer(\n", + " (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " )\n", + " (bl_1): ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (se): SEModule(\n", + " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", + " (excitation): Sequential(\n", + " (fc_reduce): Linear(in_features=128, out_features=8, bias=True)\n", + " (se_act): ReLU(inplace=True)\n", + " (fc_expand): Linear(in_features=8, out_features=128, bias=True)\n", + " (se_gate): Sigmoid()\n", + " )\n", + " )\n", + " )\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " )\n", + ")" + ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#collapse_output\n", + "mc.body.l_1" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:26:31.408148Z", + "start_time": "2021-10-11T11:26:31.285556Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 64, 32, 32])\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = mc.body.l_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:26:37.461064Z", + "start_time": "2021-10-11T11:26:37.397923Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential(\n", + " (bl_0): ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (se): SEModule(\n", + " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", + " (excitation): Sequential(\n", + " (fc_reduce): Linear(in_features=64, out_features=4, bias=True)\n", + " (se_act): ReLU(inplace=True)\n", + " (fc_expand): Linear(in_features=4, out_features=64, bias=True)\n", + " (se_gate): Sigmoid()\n", + " )\n", + " )\n", + " )\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " )\n", + " (bl_1): ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (se): SEModule(\n", + " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", + " (excitation): Sequential(\n", + " (fc_reduce): Linear(in_features=64, out_features=4, bias=True)\n", + " (se_act): ReLU(inplace=True)\n", + " (fc_expand): Linear(in_features=4, out_features=64, bias=True)\n", + " (se_gate): Sigmoid()\n", + " )\n", + " )\n", + " (sa): SimpleSelfAttention(\n", + " (conv): Conv1d(64, 64, kernel_size=(1,), stride=(1,), bias=False)\n", + " )\n", + " )\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " )\n", + ")" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#hide\n", + "mc.body.l_0" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:26:45.046280Z", + "start_time": "2021-10-11T11:26:44.925430Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 64, 32, 32])\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = mc.body.l_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:26:55.336417Z", + "start_time": "2021-10-11T11:26:55.273118Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential(\n", + " (bl_0): ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=4, bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (se): SEModule(\n", + " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", + " (excitation): Sequential(\n", + " (fc_reduce): Linear(in_features=256, out_features=16, bias=True)\n", + " (se_act): ReLU(inplace=True)\n", + " (fc_expand): Linear(in_features=16, out_features=256, bias=True)\n", + " (se_gate): Sigmoid()\n", + " )\n", + " )\n", + " )\n", + " (idconv): ConvLayer(\n", + " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " )\n", + " (bl_1): ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=4, bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (se): SEModule(\n", + " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", + " (excitation): Sequential(\n", + " (fc_reduce): Linear(in_features=256, out_features=16, bias=True)\n", + " (se_act): ReLU(inplace=True)\n", + " (fc_expand): Linear(in_features=16, out_features=256, bias=True)\n", + " (se_gate): Sigmoid()\n", + " )\n", + " )\n", + " (sa): SimpleSelfAttention(\n", + " (conv): Conv1d(256, 256, kernel_size=(1,), stride=(1,), bias=False)\n", + " )\n", + " )\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " )\n", + ")" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#hide\n", + "mc.groups = 4\n", + "mc.expansion = 4\n", + "mc.body.l_0" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:27:02.075088Z", + "start_time": "2021-10-11T11:27:01.832044Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 256, 32, 32])\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = mc.body.l_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 256, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:27:11.968760Z", + "start_time": "2021-10-11T11:27:11.911614Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Sequential(\n", + " (bl_0): ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64, bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (se): SEModule(\n", + " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", + " (excitation): Sequential(\n", + " (fc_reduce): Linear(in_features=256, out_features=16, bias=True)\n", + " (se_act): ReLU(inplace=True)\n", + " (fc_expand): Linear(in_features=16, out_features=256, bias=True)\n", + " (se_gate): Sigmoid()\n", + " )\n", + " )\n", + " )\n", + " (idconv): ConvLayer(\n", + " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " )\n", + " (bl_1): ResBlock(\n", + " (convs): Sequential(\n", + " (conv_0): ConvLayer(\n", + " (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_1): ConvLayer(\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64, bias=False)\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (conv_2): ConvLayer(\n", + " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " )\n", + " (se): SEModule(\n", + " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", + " (excitation): Sequential(\n", + " (fc_reduce): Linear(in_features=256, out_features=16, bias=True)\n", + " (se_act): ReLU(inplace=True)\n", + " (fc_expand): Linear(in_features=16, out_features=256, bias=True)\n", + " (se_gate): Sigmoid()\n", + " )\n", + " )\n", + " (sa): SimpleSelfAttention(\n", + " (conv): Conv1d(256, 256, kernel_size=(1,), stride=(1,), bias=False)\n", + " )\n", + " )\n", + " (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n", + " )\n", + ")" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#hide\n", + "mc.groups = 1\n", + "mc.dw = True\n", + "mc.expansion = 4\n", + "mc.body.l_0" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:27:18.865222Z", + "start_time": "2021-10-11T11:27:18.631385Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 256, 32, 32])\n" + ] + } + ], + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = mc.body.l_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 256, 32, 32]), f\"size\"" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:27:26.863536Z", + "start_time": "2021-10-11T11:27:26.859027Z" + } + }, + "outputs": [], + "source": [ + "#hide\n", + "mc.groups = 1\n", + "mc.dw = 0\n", + "mc.expansion = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## end\n", + "model_constructor\n", + "by ayasyrev" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "460c8d17e5de1304fcc9388854d8b1e7fdd10d3c58b2d7b68fabbdff2124405d" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Nbs/01_activations.ipynb b/Nbs/01_activations.ipynb index 690dbe2..cbb9b43 100644 --- a/Nbs/01_activations.ipynb +++ b/Nbs/01_activations.ipynb @@ -2,33 +2,42 @@ "cells": [ { "cell_type": "markdown", + "metadata": {}, "source": [ "# Activations functions.\n", "\n", "> Activations functions. Set of act_fn." - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Activation functions, forked from https://github.com/rwightman/pytorch-image-models/timm/models/layers/activations.py" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Mish: Self Regularized \n", "Non-Monotonic Activation Function \n", "https://github.com/digantamisra98/Mish \n", - "fastai forum discussion https://forums.fast.ai/t/meet-mish-new-activation-function-possible-successor-to-relu " - ], - "metadata": {} + "fastai forum discussion https://forums.fast.ai/t/meet-mish-new-activation-function-possible-successor-to-relu \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Mish is in Pytorch from version 1.9. Use this version! " + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# hide\n", "# forked from https://github.com/rwightman/pytorch-image-models/timm/models/layers/activations.py\n", @@ -36,20 +45,20 @@ "import torch\n", "from torch import nn as nn\n", "from torch.nn import functional as F" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Mish" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "def mish(x, inplace: bool = False):\n", " \"\"\"Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681\n", @@ -66,20 +75,20 @@ "\n", " def forward(self, x):\n", " return mish(x)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## MishJit" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "@torch.jit.script\n", "def mish_jit(x, _inplace: bool = False):\n", @@ -97,20 +106,20 @@ "\n", " def forward(self, x):\n", " return mish_jit(x)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## MishJitMe - memory-efficient." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "@torch.jit.script\n", "def mish_jit_fwd(x):\n", @@ -151,20 +160,20 @@ "\n", " def forward(self, x):\n", " return MishJitAutoFn.apply(x)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## HardMishJit" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "@torch.jit.script\n", "def hard_mish_jit(x, inplace: bool = False):\n", @@ -185,20 +194,20 @@ "\n", " def forward(self, x):\n", " return hard_mish_jit(x)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## HardMishJitMe - memory efficient." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "@torch.jit.script\n", "def hard_mish_jit_fwd(x):\n", @@ -242,28 +251,26 @@ "\n", " def forward(self, x):\n", " return HardMishJitAutoFn.apply(x)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "#hide\n", "act_fn = Mish(inplace=True)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# end\n", "model_constructor\n", "by ayasyrev" - ], - "metadata": {} + ] } ], "metadata": { @@ -300,4 +307,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/Nbs/04_YaResNet.ipynb b/Nbs/04_YaResNet.ipynb index 99406d8..feb5443 100644 --- a/Nbs/04_YaResNet.ipynb +++ b/Nbs/04_YaResNet.ipynb @@ -2,59 +2,137 @@ "cells": [ { "cell_type": "markdown", + "metadata": {}, "source": [ "# YaResNet.\n", "\n", "> Yet Another ResNet model." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:34:28.083063Z", + "start_time": "2021-10-11T11:34:27.356328Z" + } + }, + "outputs": [], "source": [ "#hide\n", - "# from nbdev.showdoc import *\n", - "# from fastcore.test import *\n", "import torch\n", "import torch.nn as nn\n", - "from functools import partial\n", - "from collections import OrderedDict\n", - "from model_constructor.layers import SEBlock, ConvLayer, act_fn, noop, SimpleSelfAttention\n", - "from model_constructor.net import Net\n", - "from model_constructor.activations import Mish" - ], + "\n", + "from nbdev.showdoc import show_doc\n", + "from IPython.display import Markdown, display" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:34:28.086847Z", + "start_time": "2021-10-11T11:34:28.084337Z" + } + }, "outputs": [], - "metadata": {} + "source": [ + "# hide\n", + "def print_doc(func_name):\n", + " doc = show_doc(func_name, title_level=4, disp=False)\n", + " display(Markdown(doc))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:34:32.722636Z", + "start_time": "2021-10-11T11:34:32.717788Z" + } + }, + "outputs": [], + "source": [ + "#hide\n", + "# from nbdev.showdoc import *\n", + "# from fastcore.test import *\n", + "# import torch\n", + "# import torch.nn as nn\n", + "# from functools import partial\n", + "# from collections import OrderedDict\n", + "# from model_constructor.layers import SEBlock, ConvLayer, act_fn, noop, SimpleSelfAttention\n", + "# from model_constructor.net import Net\n", + "# from model_constructor.activations import Mish" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## YaResBlock" - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:34:54.505136Z", + "start_time": "2021-10-11T11:34:54.499880Z" + } + }, + "outputs": [], "source": [ "#hide\n", "from model_constructor.yaresnet import YaResBlock" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 3, - "source": [ - "#collapse_output\n", - "bl = YaResBlock(1,64,64,sa=True)\n", - "bl" + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:34:57.272993Z", + "start_time": "2021-10-11T11:34:57.253595Z" + } + }, + "outputs": [ + { + "data": { + "text/markdown": [ + "

class YaResBlock[source]

\n", + "\n", + "> YaResBlock(**`expansion`**, **`ni`**, **`nh`**, **`stride`**=*`1`*, **`conv_layer`**=*`ConvLayer`*, **`act_fn`**=*`ReLU(inplace=True)`*, **`zero_bn`**=*`True`*, **`bn_1st`**=*`True`*, **`pool`**=*`AvgPool2d(kernel_size=2, stride=2, padding=0)`*, **`sa`**=*`False`*, **`sym`**=*`False`*, **`groups`**=*`1`*, **`dw`**=*`False`*, **`div_groups`**=*`None`*, **`se_module`**=*`SEModule`*, **`se`**=*`False`*, **`se_reduction`**=*`16`*) :: `Module`\n", + "\n", + "YaResBlock. Reduce by pool instead of stride 2" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } ], + "source": [ + "#hide_input\n", + "print_doc(YaResBlock)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:07.149507Z", + "start_time": "2021-10-11T11:35:07.116952Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "YaResBlock(\n", @@ -76,45 +154,55 @@ ")" ] }, + "execution_count": 6, "metadata": {}, - "execution_count": 3 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "bl = YaResBlock(1,64,64,sa=True)\n", + "bl" + ] }, { "cell_type": "code", - "execution_count": 4, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 64, 32, 32)\n", - "y = bl(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" - ], + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:08.094713Z", + "start_time": "2021-10-11T11:35:08.048838Z" + } + }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 64, 32, 32])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = bl(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] }, { "cell_type": "code", - "execution_count": 5, - "source": [ - "#collapse_output\n", - "bl = YaResBlock(1,64,64,se=True)\n", - "bl" - ], + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:11.146707Z", + "start_time": "2021-10-11T11:35:11.139388Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "YaResBlock(\n", @@ -128,13 +216,13 @@ " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", - " (se): SEBlock(\n", + " (se): SEModule(\n", " (squeeze): AdaptiveAvgPool2d(output_size=1)\n", " (excitation): Sequential(\n", " (fc_reduce): Linear(in_features=64, out_features=4, bias=True)\n", " (se_act): ReLU(inplace=True)\n", " (fc_expand): Linear(in_features=4, out_features=64, bias=True)\n", - " (sigmoid): Sigmoid()\n", + " (se_gate): Sigmoid()\n", " )\n", " )\n", " )\n", @@ -142,45 +230,55 @@ ")" ] }, + "execution_count": 8, "metadata": {}, - "execution_count": 5 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "bl = YaResBlock(1,64,64,se=True)\n", + "bl" + ] }, { "cell_type": "code", - "execution_count": 6, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 64, 32, 32)\n", - "y = bl(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" - ], + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:12.172367Z", + "start_time": "2021-10-11T11:35:12.134552Z" + } + }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 64, 32, 32])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = bl(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] }, { "cell_type": "code", - "execution_count": 7, - "source": [ - "#collapse_output\n", - "bl = YaResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False)\n", - "bl" - ], + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:13.997558Z", + "start_time": "2021-10-11T11:35:13.971785Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "YaResBlock(\n", @@ -209,45 +307,55 @@ ")" ] }, + "execution_count": 10, "metadata": {}, - "execution_count": 7 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "bl = YaResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False)\n", + "bl" + ] }, { "cell_type": "code", - "execution_count": 8, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 256, 32, 32)\n", - "y = bl(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" - ], + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:14.377916Z", + "start_time": "2021-10-11T11:35:14.291710Z" + } + }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 512, 16, 16])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 256, 32, 32)\n", + "y = bl(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" + ] }, { "cell_type": "code", - "execution_count": 9, - "source": [ - "#collapse_output\n", - "bl = YaResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False, groups=4)\n", - "bl" - ], + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:14.574673Z", + "start_time": "2021-10-11T11:35:14.568217Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "YaResBlock(\n", @@ -276,45 +384,55 @@ ")" ] }, + "execution_count": 12, "metadata": {}, - "execution_count": 9 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "bl = YaResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False, groups=4)\n", + "bl" + ] }, { "cell_type": "code", - "execution_count": 10, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 256, 32, 32)\n", - "y = bl(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" - ], + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:17.617154Z", + "start_time": "2021-10-11T11:35:17.513494Z" + } + }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 512, 16, 16])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 256, 32, 32)\n", + "y = bl(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" + ] }, { "cell_type": "code", - "execution_count": 11, - "source": [ - "#collapse_output\n", - "bl = YaResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False, div_groups=4)\n", - "bl" - ], + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:17.793398Z", + "start_time": "2021-10-11T11:35:17.788172Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "YaResBlock(\n", @@ -343,45 +461,55 @@ ")" ] }, + "execution_count": 14, "metadata": {}, - "execution_count": 11 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "bl = YaResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False, div_groups=4)\n", + "bl" + ] }, { "cell_type": "code", - "execution_count": 12, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 256, 32, 32)\n", - "y = bl(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" - ], + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:18.125027Z", + "start_time": "2021-10-11T11:35:18.028738Z" + } + }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 512, 16, 16])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 256, 32, 32)\n", + "y = bl(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" + ] }, { "cell_type": "code", - "execution_count": 13, - "source": [ - "#collapse_output\n", - "bl = YaResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False, dw=True)\n", - "bl" - ], + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:20.338920Z", + "start_time": "2021-10-11T11:35:20.331674Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "YaResBlock(\n", @@ -410,106 +538,133 @@ ")" ] }, + "execution_count": 16, "metadata": {}, - "execution_count": 13 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "bl = YaResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False, dw=True)\n", + "bl" + ] }, { "cell_type": "code", - "execution_count": 14, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 256, 32, 32)\n", - "y = bl(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" - ], + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:35:20.744351Z", + "start_time": "2021-10-11T11:35:20.645168Z" + } + }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 512, 16, 16])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 256, 32, 32)\n", + "y = bl(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## YaResNet constructor." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 15, - "source": [ - "yaresnet = Net(block=YaResBlock, stem_sizes = [3, 32, 64, 64], name='YaResNet')" - ], + "execution_count": 19, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:36:05.397700Z", + "start_time": "2021-10-11T11:36:05.393021Z" + } + }, "outputs": [], - "metadata": {} + "source": [ + "from model_constructor import ModelConstructor\n", + "yaresnet = ModelConstructor(block=YaResBlock, stem_sizes = [3, 32, 64, 64], name='YaResNet')" + ] }, { "cell_type": "code", - "execution_count": 16, - "source": [ - "yaresnet" - ], + "execution_count": 20, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:36:08.676113Z", + "start_time": "2021-10-11T11:36:08.670906Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "YaResNet constructor\n", " c_in: 3, c_out: 1000\n", - " expansion: 1, groups: 1, dw: False\n", + " expansion: 1, groups: 1, dw: False, div_groups: None\n", " sa: False, se: False\n", - " stem sizes: [3, 32, 64, 64], stide on 0\n", + " stem sizes: [3, 32, 64, 64], stride on 0\n", " body sizes [64, 128, 256, 512]\n", " layers: [2, 2, 2, 2]" ] }, + "execution_count": 20, "metadata": {}, - "execution_count": 16 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "yaresnet" + ] }, { "cell_type": "code", - "execution_count": 17, - "source": [ - "yaresnet.block_sizes, yaresnet.layers" - ], + "execution_count": 21, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:36:15.308029Z", + "start_time": "2021-10-11T11:36:15.302140Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "([64, 64, 128, 256, 512], [2, 2, 2, 2])" ] }, + "execution_count": 21, "metadata": {}, - "execution_count": 17 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "yaresnet.block_sizes, yaresnet.layers" + ] }, { "cell_type": "code", - "execution_count": 18, - "source": [ - "#collapse_output\n", - "yaresnet.stem" - ], + "execution_count": 22, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:36:16.876151Z", + "start_time": "2021-10-11T11:36:16.869821Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -532,52 +687,62 @@ ")" ] }, + "execution_count": 22, "metadata": {}, - "execution_count": 18 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "yaresnet.stem" + ] }, { "cell_type": "code", - "execution_count": 19, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 3, 128, 128)\n", - "y = yaresnet.stem(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" - ], + "execution_count": 23, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:36:19.758319Z", + "start_time": "2021-10-11T11:36:19.616433Z" + } + }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 64, 32, 32])\n" ] }, { - "output_type": "stream", "name": "stderr", + "output_type": "stream", "text": [ - "/home/jzz/anaconda3/envs/mc_dev/lib/python3.9/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1623448255797/work/c10/core/TensorImpl.h:1156.)\n", + "/home/jzz/anaconda3/envs/pt_19/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1623448278899/work/c10/core/TensorImpl.h:1156.)\n", " return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 3, 128, 128)\n", + "y = yaresnet.stem(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] }, { "cell_type": "code", - "execution_count": 20, - "source": [ - "#collapse_output\n", - "yaresnet.body" - ], + "execution_count": 24, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:36:25.645719Z", + "start_time": "2021-10-11T11:36:25.581093Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -719,22 +884,27 @@ ")" ] }, + "execution_count": 24, "metadata": {}, - "execution_count": 20 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "yaresnet.body" + ] }, { "cell_type": "code", - "execution_count": 21, - "source": [ - "#collapse_output\n", - "yaresnet.head" - ], + "execution_count": 25, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:36:26.325358Z", + "start_time": "2021-10-11T11:36:26.317998Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -744,34 +914,38 @@ ")" ] }, + "execution_count": 25, "metadata": {}, - "execution_count": 21 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "yaresnet.head" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Lots of experiments showed that it worth trying Mish activation function." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 22, - "source": [ - "#collapse_output\n", - "yaresnet.act_fn = Mish()\n", - "yaresnet()" - ], + "execution_count": 27, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:36:53.086959Z", + "start_time": "2021-10-11T11:36:52.966174Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", - " model YaResNet\n", + " YaResNet\n", " (stem): Sequential(\n", " (conv_0): ConvLayer(\n", " (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", @@ -935,145 +1109,89 @@ ")" ] }, + "execution_count": 27, "metadata": {}, - "execution_count": 22 + "output_type": "execute_result" } ], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": null, - "source": [], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "markdown", - "source": [ - "### Predefined YResNet models." - ], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 23, - "source": [ - "yaresnet_parameters = {'block': YaResBlock, 'stem_sizes': [3, 32, 64, 64], 'act_fn': Mish(), 'stem_stride_on': 1}\n", - "yaresnet34 = partial(Net, name='YaResnet34', expansion=1, layers=[3, 4, 6, 3], **yaresnet_parameters)\n", - "yaresnet50 = partial(Net, name='YaResnet50', expansion=4, layers=[3, 4, 6, 3], **yaresnet_parameters)" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 24, "source": [ - "model = yaresnet50(c_out=10)" - ], - "outputs": [], - "metadata": {} + "#collapse_output\n", + "yaresnet.act_fn = torch.nn.Mish()\n", + "yaresnet()" + ] }, { "cell_type": "code", - "execution_count": 25, - "source": [ - "model" - ], - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "YaResnet50 constructor\n", - " c_in: 3, c_out: 10\n", - " expansion: 4, groups: 1, dw: False\n", - " sa: False, se: False\n", - " stem sizes: [3, 32, 64, 64], stide on 1\n", - " body sizes [64, 128, 256, 512]\n", - " layers: [3, 4, 6, 3]" - ] - }, - "metadata": {}, - "execution_count": 25 + "execution_count": 30, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:38:07.756257Z", + "start_time": "2021-10-11T11:38:07.295368Z" } - ], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 26, - "source": [ - "model.c_out, model.layers" - ], + }, "outputs": [ { - "output_type": "execute_result", - "data": { - "text/plain": [ - "(10, [3, 4, 6, 3])" - ] - }, - "metadata": {}, - "execution_count": 26 + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 1000])\n" + ] } ], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 27, "source": [ "#hide\n", "bs_test = 16\n", "xb = torch.randn(bs_test, 3, 128, 128)\n", - "y = model()(xb)\n", + "y = yaresnet()(xb)\n", "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 10]), f\"size\"" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "torch.Size([16, 10])\n" - ] - } - ], - "metadata": {} + "assert y.shape == torch.Size([bs_test, 1000]), f\"size\"" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# end\n", "model_constructor\n", "by ayasyrev" - ], - "metadata": {} + ] } ], "metadata": { + "interpreter": { + "hash": "460c8d17e5de1304fcc9388854d8b1e7fdd10d3c58b2d7b68fabbdff2124405d" + }, "kernelspec": { - "name": "python3", - "display_name": "Python 3.9.6 64-bit ('mc_dev': conda)" + "display_name": "Python 3", + "language": "python", + "name": "python3" }, "language_info": { - "name": "python", - "version": "3.9.6", - "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, - "pygments_lexer": "ipython3", + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", "nbconvert_exporter": "python", - "file_extension": ".py" + "pygments_lexer": "ipython3", + "version": "3.8.10" }, - "interpreter": { - "hash": "460c8d17e5de1304fcc9388854d8b1e7fdd10d3c58b2d7b68fabbdff2124405d" + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true } }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/Nbs/00_Net.ipynb b/Nbs/09_Net.ipynb similarity index 95% rename from Nbs/00_Net.ipynb rename to Nbs/09_Net.ipynb index 735bf87..49a14c5 100644 --- a/Nbs/00_Net.ipynb +++ b/Nbs/09_Net.ipynb @@ -2,25 +2,29 @@ "cells": [ { "cell_type": "markdown", + "metadata": {}, "source": [ - "# Model constructor class.\n", + "# First version. Net.\n", "\n", - "> Create and tune pytorch model." - ], - "metadata": {} + "> First version of constructor." + ] }, { "cell_type": "code", "execution_count": 1, + "metadata": { + "code_folding": [] + }, + "outputs": [], "source": [ "#hide" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 2, + "metadata": {}, + "outputs": [], "source": [ "#hide\n", "import torch\n", @@ -31,13 +35,13 @@ "\n", "# from fastcore.test import *\n", "from dataclasses import dataclass, asdict" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 3, + "metadata": {}, + "outputs": [], "source": [ "# hide\n", "from collections import OrderedDict\n", @@ -45,32 +49,32 @@ "\n", "\n", "from model_constructor.layers import ConvLayer, Flatten, SEBlock, SimpleSelfAttention, noop" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 4, + "metadata": {}, + "outputs": [], "source": [ "# hide\n", "def print_doc(func_name):\n", " doc = show_doc(func_name, title_level=4, disp=False)\n", " display(Markdown(doc))" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# Utils" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 5, + "metadata": {}, + "outputs": [], "source": [ "act_fn = nn.ReLU(inplace=True)\n", "\n", @@ -82,20 +86,20 @@ " nn.init.kaiming_normal_(module.weight)\n", " for layer in module.children():\n", " init_cnn(layer)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# ResBlock" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 6, + "metadata": {}, + "outputs": [], "source": [ "\n", "\n", @@ -133,19 +137,14 @@ "\n", " def forward(self, x):\n", " return self.act_fn(self.convs(x) + self.idconv(self.pool(x)))" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 7, - "source": [ - "ResBlock(1,64,64,sa=True)" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "ResBlock(\n", @@ -167,21 +166,21 @@ ")" ] }, + "execution_count": 7, "metadata": {}, - "execution_count": 7 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "ResBlock(1,64,64,sa=True)" + ] }, { "cell_type": "code", "execution_count": 8, - "source": [ - "ResBlock(4,64,64,sa=True, dw=True)" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "ResBlock(\n", @@ -208,21 +207,21 @@ ")" ] }, + "execution_count": 8, "metadata": {}, - "execution_count": 8 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "ResBlock(4,64,64,sa=True, dw=True)" + ] }, { "cell_type": "code", "execution_count": 9, - "source": [ - "ResBlock(4,64,64,sa=True, groups=4)" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "ResBlock(\n", @@ -249,21 +248,21 @@ ")" ] }, + "execution_count": 9, "metadata": {}, - "execution_count": 9 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "ResBlock(4,64,64,sa=True, groups=4)" + ] }, { "cell_type": "code", "execution_count": 10, - "source": [ - "ResBlock(2,64,64,act_fn=nn.LeakyReLU(), bn_1st=False)" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "ResBlock(\n", @@ -287,21 +286,21 @@ ")" ] }, + "execution_count": 10, "metadata": {}, - "execution_count": 10 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "ResBlock(2,64,64,act_fn=nn.LeakyReLU(), bn_1st=False)" + ] }, { "cell_type": "code", "execution_count": 11, - "source": [ - "ResBlock(2, 64, 64, sa=True, se=True)" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "ResBlock(\n", @@ -337,30 +336,35 @@ ")" ] }, + "execution_count": 11, "metadata": {}, - "execution_count": 11 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "ResBlock(2, 64, 64, sa=True, se=True)" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# NewResBlock" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "NewResBlock now is YaResBlock - Yet Another ResNet Block! It is now at model_constructor.yaresnet.\n", "Here i left old name for compatibility with existing Notebooks." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 12, + "metadata": {}, + "outputs": [], "source": [ "# NewResBlock now is YaResBlock - Yet Another ResNet Block! It is now at model_constructor.yaresnet.\n", "class NewResBlock(nn.Module):\n", @@ -398,19 +402,14 @@ " def forward(self, x):\n", " o = self.reduce(x)\n", " return self.merge(self.convs(o) + self.idconv(o))" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 13, - "source": [ - "NewResBlock(4, 64, 128, dw=1)" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "NewResBlock(\n", @@ -438,34 +437,32 @@ ")" ] }, + "execution_count": 13, "metadata": {}, - "execution_count": 13 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "NewResBlock(4, 64, 128, dw=1)" + ] }, { "cell_type": "code", "execution_count": 14, + "metadata": {}, + "outputs": [], "source": [ "#hide\n", "# from model_constructor.yaresnet import YaResBlock\n", "# NewResBlock = YaResBlock" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 15, - "source": [ - "#hide\n", - "bl = NewResBlock(1,64,64,sa=True)\n", - "bl" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "NewResBlock(\n", @@ -487,45 +484,45 @@ ")" ] }, + "execution_count": 15, "metadata": {}, - "execution_count": 15 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "bl = NewResBlock(1,64,64,sa=True)\n", + "bl" + ] }, { "cell_type": "code", "execution_count": 16, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 64, 32, 32)\n", - "y = bl(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 64, 32, 32])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = bl(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] }, { "cell_type": "code", "execution_count": 17, - "source": [ - "#hide\n", - "bl = NewResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False)\n", - "bl" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "NewResBlock(\n", @@ -554,45 +551,45 @@ ")" ] }, + "execution_count": 17, "metadata": {}, - "execution_count": 17 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "bl = NewResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False)\n", + "bl" + ] }, { "cell_type": "code", "execution_count": 18, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 256, 32, 32)\n", - "y = bl(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 512, 16, 16])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 256, 32, 32)\n", + "y = bl(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" + ] }, { "cell_type": "code", "execution_count": 19, - "source": [ - "#hide\n", - "bl = NewResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False, groups=4)\n", - "bl" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "NewResBlock(\n", @@ -621,44 +618,51 @@ ")" ] }, + "execution_count": 19, "metadata": {}, - "execution_count": 19 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "bl = NewResBlock(4,64,128,stride=2,act_fn=nn.LeakyReLU(), bn_1st=False, groups=4)\n", + "bl" + ] }, { "cell_type": "code", "execution_count": 20, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 256, 32, 32)\n", - "y = bl(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 512, 16, 16])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 256, 32, 32)\n", + "y = bl(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# Stem, Body, Head" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 21, + "metadata": {}, + "outputs": [], "source": [ "def _make_stem(self):\n", " stem = [(f\"conv_{i}\", self.conv_layer(self.stem_sizes[i], self.stem_sizes[i + 1],\n", @@ -670,13 +674,13 @@ " if self.stem_bn_end:\n", " stem.append(('norm', self.norm(self.stem_sizes[-1])))\n", " return nn.Sequential(OrderedDict(stem))" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 22, + "metadata": {}, + "outputs": [], "source": [ "def _make_layer(self, expansion, ni, nf, blocks, stride, sa):\n", " layers = [(f\"bl_{i}\", self.block(expansion, ni if i == 0 else nf, nf,\n", @@ -686,13 +690,13 @@ " dw=self.dw, se=self.se))\n", " for i in range(blocks)]\n", " return nn.Sequential(OrderedDict(layers))" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 23, + "metadata": {}, + "outputs": [], "source": [ "def _make_body(self):\n", " blocks = [(f\"l_{i}\", self._make_layer(self, self.expansion,\n", @@ -701,33 +705,33 @@ " sa=self.sa if i == 0 else False))\n", " for i, l in enumerate(self.layers)]\n", " return nn.Sequential(OrderedDict(blocks))" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 24, + "metadata": {}, + "outputs": [], "source": [ "def _make_head(self):\n", " head = [('pool', nn.AdaptiveAvgPool2d(1)),\n", " ('flat', Flatten()),\n", " ('fc', nn.Linear(self.block_sizes[-1] * self.expansion, self.c_out))]\n", " return nn.Sequential(OrderedDict(head))" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# Net - Model Constructor." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 25, + "metadata": {}, + "outputs": [], "source": [ "class Net():\n", " \"\"\"Model constructor. As default - xresnet18\"\"\"\n", @@ -793,20 +797,14 @@ " f\" stem sizes: {self.stem_sizes}, stide on {self.stem_stride_on}\\n\"\n", " f\" body sizes {self._block_sizes}\\n\"\n", " f\" layers: {self.layers}\")" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 26, - "source": [ - "model = Net()\n", - "model" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Net constructor\n", @@ -818,62 +816,62 @@ " layers: [2, 2, 2, 2]" ] }, + "execution_count": 26, "metadata": {}, - "execution_count": 26 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "model = Net()\n", + "model" + ] }, { "cell_type": "code", "execution_count": 27, - "source": [ - "model._block_sizes" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "[64, 128, 256, 512]" ] }, + "execution_count": 27, "metadata": {}, - "execution_count": 27 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "model._block_sizes" + ] }, { "cell_type": "code", "execution_count": 28, - "source": [ - "model.block_sizes" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "[64, 64, 128, 256, 512]" ] }, + "execution_count": 28, "metadata": {}, - "execution_count": 28 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "model.block_sizes" + ] }, { "cell_type": "code", "execution_count": 29, - "source": [ - "model._block_sizes = [128, 256, 512, 1024]\n", - "model" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Net constructor\n", @@ -885,43 +883,42 @@ " layers: [2, 2, 2, 2]" ] }, + "execution_count": 29, "metadata": {}, - "execution_count": 29 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "model._block_sizes = [128, 256, 512, 1024]\n", + "model" + ] }, { "cell_type": "code", "execution_count": 30, - "source": [ - "model.block_sizes" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "[64, 128, 256, 512, 1024]" ] }, + "execution_count": 30, "metadata": {}, - "execution_count": 30 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "model.block_sizes" + ] }, { "cell_type": "code", "execution_count": 31, - "source": [ - "#hide\n", - "model = Net(stem_sizes=[3,32,32,64])\n", - "model" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Net constructor\n", @@ -933,62 +930,63 @@ " layers: [2, 2, 2, 2]" ] }, + "execution_count": 31, "metadata": {}, - "execution_count": 31 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model = Net(stem_sizes=[3,32,32,64])\n", + "model" + ] }, { "cell_type": "code", "execution_count": 32, - "source": [ - "model.block_sizes" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "[64, 64, 128, 256, 512]" ] }, + "execution_count": 32, "metadata": {}, - "execution_count": 32 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "model.block_sizes" + ] }, { "cell_type": "code", "execution_count": 33, + "metadata": {}, + "outputs": [], "source": [ "model = Net()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 34, + "metadata": {}, + "outputs": [], "source": [ "#hide\n", "# model.block_sizes = [64, 128, 256, 512] # wrong way --> use _block_sizes\n", "# model" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 35, - "source": [ - "#collapse_output\n", - "model.stem" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -1011,23 +1009,22 @@ ")" ] }, + "execution_count": 35, "metadata": {}, - "execution_count": 35 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "model.stem" + ] }, { "cell_type": "code", "execution_count": 36, - "source": [ - "#collapse_output\n", - "model.stem_stride_on = 1\n", - "model.stem" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -1050,80 +1047,81 @@ ")" ] }, + "execution_count": 36, "metadata": {}, - "execution_count": 36 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "model.stem_stride_on = 1\n", + "model.stem" + ] }, { "cell_type": "code", "execution_count": 37, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 3, 128, 128)\n", - "y = model.stem(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 64, 32, 32])\n" ] }, { - "output_type": "stream", "name": "stderr", + "output_type": "stream", "text": [ "/home/jzz/anaconda3/envs/mc_dev/lib/python3.9/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1623448255797/work/c10/core/TensorImpl.h:1156.)\n", " return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 3, 128, 128)\n", + "y = model.stem(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] }, { "cell_type": "code", "execution_count": 38, + "metadata": {}, + "outputs": [], "source": [ "model.bn_1st = False" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 39, + "metadata": {}, + "outputs": [], "source": [ "model.act_fn =nn.LeakyReLU(inplace=True)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 40, + "metadata": {}, + "outputs": [], "source": [ "model.sa = True\n", "model.se = True" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 41, - "source": [ - "#collapse_output\n", - "model.body.l_1" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -1181,44 +1179,44 @@ ")" ] }, + "execution_count": 41, "metadata": {}, - "execution_count": 41 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "model.body.l_1" + ] }, { "cell_type": "code", "execution_count": 42, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 64, 32, 32)\n", - "y = model.body.l_0(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 64, 32, 32])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = model.body.l_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] }, { "cell_type": "code", "execution_count": 43, - "source": [ - "#hide\n", - "model.body.l_0" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -1274,46 +1272,44 @@ ")" ] }, + "execution_count": 43, "metadata": {}, - "execution_count": 43 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model.body.l_0" + ] }, { "cell_type": "code", "execution_count": 44, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 64, 32, 32)\n", - "y = model.body.l_0(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 64, 32, 32])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = model.body.l_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 64, 32, 32]), f\"size\"" + ] }, { "cell_type": "code", - "execution_count": 45, - "source": [ - "#hide\n", - "model.groups = 4\n", - "model.expansion = 4\n", - "model.body.l_0" - ], + "execution_count": 45, + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -1383,47 +1379,46 @@ ")" ] }, + "execution_count": 45, "metadata": {}, - "execution_count": 45 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model.groups = 4\n", + "model.expansion = 4\n", + "model.body.l_0" + ] }, { "cell_type": "code", "execution_count": 46, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 64, 32, 32)\n", - "y = model.body.l_0(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 256, 32, 32]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 256, 32, 32])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = model.body.l_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 256, 32, 32]), f\"size\"" + ] }, { "cell_type": "code", "execution_count": 47, - "source": [ - "#hide\n", - "model.groups = 1\n", - "model.dw = True\n", - "model.expansion = 4\n", - "model.body.l_0" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -1493,87 +1488,90 @@ ")" ] }, + "execution_count": 47, "metadata": {}, - "execution_count": 47 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model.groups = 1\n", + "model.dw = True\n", + "model.expansion = 4\n", + "model.body.l_0" + ] }, { "cell_type": "code", "execution_count": 48, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 64, 32, 32)\n", - "y = model.body.l_0(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 256, 32, 32]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 256, 32, 32])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = model.body.l_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 256, 32, 32]), f\"size\"" + ] }, { "cell_type": "code", "execution_count": 49, + "metadata": {}, + "outputs": [], "source": [ "#hide\n", "model.groups = 1\n", "model.dw = 0\n", "model.expansion = 1" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 50, + "metadata": {}, + "outputs": [], "source": [ "model.block = NewResBlock" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 51, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 64, 32, 32)\n", - "y = model.body.l_1.bl_0(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 128, 16, 16]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 128, 16, 16])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 64, 32, 32)\n", + "y = model.body.l_1.bl_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 128, 16, 16]), f\"size\"" + ] }, { "cell_type": "code", "execution_count": 52, - "source": [ - "#hide\n", - "model.body.l_1.bl_0" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "NewResBlock(\n", @@ -1606,42 +1604,42 @@ ")" ] }, + "execution_count": 52, "metadata": {}, - "execution_count": 52 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model.body.l_1.bl_0" + ] }, { "cell_type": "code", "execution_count": 53, + "metadata": {}, + "outputs": [], "source": [ "# model = Net(expansion=4)\n", "model.expansion = 4" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 54, + "metadata": {}, + "outputs": [], "source": [ "#hide\n", "model.stem_bn_end = True" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 55, - "source": [ - "#hide\n", - "model.stem" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -1664,22 +1662,22 @@ ")" ] }, + "execution_count": 55, "metadata": {}, - "execution_count": 55 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model.stem" + ] }, { "cell_type": "code", "execution_count": 56, - "source": [ - "#hide\n", - "model.body.l_1.bl_0" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "NewResBlock(\n", @@ -1717,64 +1715,64 @@ ")" ] }, + "execution_count": 56, "metadata": {}, - "execution_count": 56 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model.body.l_1.bl_0" + ] }, { "cell_type": "code", "execution_count": 57, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 256, 32, 32)\n", - "y = model.body.l_1.bl_0(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 512, 16, 16])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 256, 32, 32)\n", + "y = model.body.l_1.bl_0(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 512, 16, 16]), f\"size\"" + ] }, { "cell_type": "code", "execution_count": 58, + "metadata": {}, + "outputs": [], "source": [ "#hide\n", "model.stem_bn_end = False" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 59, + "metadata": {}, + "outputs": [], "source": [ "#collapse_input\n", "m = model()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 60, - "source": [ - "#hide\n", - "m" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -2061,22 +2059,22 @@ ")" ] }, + "execution_count": 60, "metadata": {}, - "execution_count": 60 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "m" + ] }, { "cell_type": "code", "execution_count": 61, - "source": [ - "#collapse_output\n", - "m.body" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -2337,37 +2335,43 @@ ")" ] }, + "execution_count": 61, "metadata": {}, - "execution_count": 61 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#collapse_output\n", + "m.body" + ] }, { "cell_type": "code", "execution_count": 62, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 3, 128, 128)\n", - "y = m(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 1000]), f\"size expected {bs_test}, 1000\"" - ], + "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 1000])\n" ] } ], - "metadata": {} + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 3, 128, 128)\n", + "y = m(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 1000]), f\"size expected {bs_test}, 1000\"" + ] }, { "cell_type": "code", "execution_count": 63, + "metadata": {}, + "outputs": [], "source": [ "## xresnet constructor\n", "\n", @@ -2381,31 +2385,31 @@ "# # setattr(me, name, partial(Net, expansion=e, layers=l, name=name))\n", "# net34 = partial(Net, expansion=1, layers=[3, 4, 6, 3], name='xresnet34')\n", "# net50 = partial(Net, expansion=4, layers=[3, 4, 6, 3], name='xresnet50')" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 64, + "metadata": {}, + "outputs": [], "source": [ "# m = net50(c_out=10)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 65, + "metadata": {}, + "outputs": [], "source": [ "# m, m.c_out" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 66, + "metadata": {}, + "outputs": [], "source": [ "#hide\n", "xresnet34_parameters = {\n", @@ -2419,21 +2423,14 @@ " 'layers': [3, 4, 6, 3],\n", " }\n", "xresnet34 = partial(Net, **xresnet34_parameters)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 67, - "source": [ - "#hide\n", - "model34 = Net(**xresnet34_parameters)\n", - "model34" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "xresnet34 constructor\n", @@ -2445,23 +2442,23 @@ " layers: [3, 4, 6, 3]" ] }, + "execution_count": 67, "metadata": {}, - "execution_count": 67 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model34 = Net(**xresnet34_parameters)\n", + "model34" + ] }, { "cell_type": "code", "execution_count": 68, - "source": [ - "#hide\n", - "model50 = Net(**xresnet50_parameters)\n", - "model50" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "xresnet50 constructor\n", @@ -2473,33 +2470,33 @@ " layers: [3, 4, 6, 3]" ] }, + "execution_count": 68, "metadata": {}, - "execution_count": 68 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model50 = Net(**xresnet50_parameters)\n", + "model50" + ] }, { "cell_type": "code", "execution_count": 69, + "metadata": {}, + "outputs": [], "source": [ "xresnet34 = partial(Net, name='xresnet34', expansion=1, layers=[3, 4, 6, 3])\n", "xresnet50 = partial(Net, name='xresnet34', expansion=4, layers=[3, 4, 6, 3])" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 70, - "source": [ - "#hide\n", - "model34 = xresnet34()\n", - "model34" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "xresnet34 constructor\n", @@ -2511,23 +2508,23 @@ " layers: [3, 4, 6, 3]" ] }, + "execution_count": 70, "metadata": {}, - "execution_count": 70 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model34 = xresnet34()\n", + "model34" + ] }, { "cell_type": "code", "execution_count": 71, - "source": [ - "#hide\n", - "model50 = xresnet50()\n", - "model50" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "xresnet34 constructor\n", @@ -2539,34 +2536,35 @@ " layers: [3, 4, 6, 3]" ] }, + "execution_count": 71, "metadata": {}, - "execution_count": 71 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "#hide\n", + "model50 = xresnet50()\n", + "model50" + ] }, { "cell_type": "code", "execution_count": 72, + "metadata": {}, + "outputs": [], "source": [ "class XResNet34(Net):\n", " def __init__(self):\n", " super().__init__()\n", " self.layers = [3,4,6,3]" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 73, - "source": [ - "model = XResNet34()\n", - "model" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Net constructor\n", @@ -2578,115 +2576,115 @@ " layers: [3, 4, 6, 3]" ] }, + "execution_count": 73, "metadata": {}, - "execution_count": 73 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "model = XResNet34()\n", + "model" + ] }, { "cell_type": "code", "execution_count": 74, + "metadata": {}, + "outputs": [], "source": [ "from dataclasses import field" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 75, + "metadata": {}, + "outputs": [], "source": [ "@dataclass\n", "class ConfXresnet34:\n", " name: str = 'xresnet34'\n", " layers: list = field(default_factory=lambda : [3,4,6,3])" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 76, - "source": [ - "asdict(ConfXresnet34())" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "{'name': 'xresnet34', 'layers': [3, 4, 6, 3]}" ] }, + "execution_count": 76, "metadata": {}, - "execution_count": 76 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "asdict(ConfXresnet34())" + ] }, { "cell_type": "code", "execution_count": 77, + "metadata": {}, + "outputs": [], "source": [ "@dataclass\n", "class Xres50(ConfXresnet34):\n", " name = 'xresnet50'\n", " expansion: int = 4" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 78, - "source": [ - "Xres50()" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Xres50(name='xresnet34', layers=[3, 4, 6, 3], expansion=4)" ] }, + "execution_count": 78, "metadata": {}, - "execution_count": 78 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "Xres50()" + ] }, { "cell_type": "code", "execution_count": 79, - "source": [ - "asdict(Xres50())" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "{'name': 'xresnet34', 'layers': [3, 4, 6, 3], 'expansion': 4}" ] }, + "execution_count": 79, "metadata": {}, - "execution_count": 79 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "asdict(Xres50())" + ] }, { "cell_type": "code", "execution_count": 80, - "source": [ - "model = Net(asdict(Xres50()))\n", - "model" - ], + "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "{'name': 'xresnet34', 'layers': [3, 4, 6, 3], 'expansion': 4} constructor\n", @@ -2698,26 +2696,34 @@ " layers: [2, 2, 2, 2]" ] }, + "execution_count": 80, "metadata": {}, - "execution_count": 80 + "output_type": "execute_result" } ], - "metadata": {} + "source": [ + "model = Net(asdict(Xres50()))\n", + "model" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# end\n", "model_constructor\n", "by ayasyrev" - ], - "metadata": {} + ] } ], "metadata": { + "interpreter": { + "hash": "460c8d17e5de1304fcc9388854d8b1e7fdd10d3c58b2d7b68fabbdff2124405d" + }, "kernelspec": { - "name": "python3", - "display_name": "Python 3.9.6 64-bit ('mc_dev': conda)" + "display_name": "Python 3", + "language": "python", + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -2729,7 +2735,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.6" + "version": "3.8.10" }, "toc": { "base_numbering": 1, @@ -2743,11 +2749,8 @@ "toc_position": {}, "toc_section_display": true, "toc_window_display": false - }, - "interpreter": { - "hash": "460c8d17e5de1304fcc9388854d8b1e7fdd10d3c58b2d7b68fabbdff2124405d" } }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/Nbs/index.ipynb b/Nbs/index.ipynb index 1466340..887ed7c 100644 --- a/Nbs/index.ipynb +++ b/Nbs/index.ipynb @@ -3,324 +3,200 @@ { "cell_type": "code", "execution_count": 1, - "source": [ - "#hide\n", - "#from nbdev import *" - ], - "outputs": [], "metadata": { "ExecuteTime": { "end_time": "2021-08-11T15:38:05.074052Z", "start_time": "2021-08-11T15:38:05.067809Z" }, "hide_input": true - } + }, + "outputs": [], + "source": [ + "#hide\n", + "#from nbdev import *" + ] }, { "cell_type": "code", "execution_count": 2, - "source": [ - "#hide\n", - "# %reload_ext autoreload\n", - "# %autoreload 2" - ], - "outputs": [], "metadata": { "ExecuteTime": { "end_time": "2021-08-11T15:38:11.176789Z", "start_time": "2021-08-11T15:38:11.172171Z" } - } + }, + "outputs": [], + "source": [ + "#hide\n", + "# %reload_ext autoreload\n", + "# %autoreload 2" + ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T10:56:13.820684Z", + "start_time": "2021-10-11T10:56:13.816995Z" + } + }, + "outputs": [], "source": [ "#hide\n", "from functools import partial\n", "import torch" - ], - "outputs": [], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:19:55.885412Z", - "start_time": "2021-08-11T16:19:55.363348Z" - } - } + ] }, { "cell_type": "code", "execution_count": 2, - "source": [ - "#hide\n", - "# from model_constructor.layers import *" - ], - "outputs": [], "metadata": { "ExecuteTime": { "end_time": "2021-08-11T16:19:55.888391Z", "start_time": "2021-08-11T16:19:55.886710Z" } - } + }, + "outputs": [], + "source": [ + "#hide\n", + "# from model_constructor.layers import *" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "# model_constructor\n", "\n", "> Constructor to create pytorch model. " - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Install" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "`pip install model-constructor`" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Or install from repo:" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "`pip install git+https://github.com/ayasyrev/model_constructor.git`" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## How to use" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ - "First import constructor class, then create model constructor oject.\n", + "First import constructor class, then create model constructor object.\n", "\n", "Now you can change every part of model." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 3, - "source": [ - "from model_constructor.net import Net" - ], - "outputs": [], + "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:00.762994Z", - "start_time": "2021-08-11T16:20:00.756490Z" + "end_time": "2021-10-11T10:53:12.809650Z", + "start_time": "2021-10-11T10:53:12.805555Z" } - } - }, - { - "cell_type": "code", - "execution_count": 4, - "source": [ - "model = Net()" - ], + }, "outputs": [], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:01.011949Z", - "start_time": "2021-08-11T16:20:01.008000Z" - } - } + "source": [ + "from model_constructor import ModelConstructor" + ] }, { "cell_type": "code", - "execution_count": 5, - "source": [ - "model" - ], - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "Net constructor\n", - " c_in: 3, c_out: 1000\n", - " expansion: 1, groups: 1, dw: False\n", - " sa: False, se: False\n", - " stem sizes: [3, 32, 32, 64], stide on 0\n", - " body sizes [64, 128, 256, 512]\n", - " layers: [2, 2, 2, 2]" - ] - }, - "metadata": {}, - "execution_count": 5 - } - ], + "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:02.893047Z", - "start_time": "2021-08-11T16:20:02.884697Z" + "end_time": "2021-10-11T10:53:20.618958Z", + "start_time": "2021-10-11T10:53:20.613551Z" } - } - }, - { - "cell_type": "markdown", + }, + "outputs": [], "source": [ - "Now we have model consructor, default setting as xresnet18. And we can get model after call it." - ], - "metadata": {} + "mc = ModelConstructor()" + ] }, { "cell_type": "code", - "execution_count": 6, - "source": [ - "model.c_in" - ], - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "3" - ] - }, - "metadata": {}, - "execution_count": 6 - } - ], + "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:03.281931Z", - "start_time": "2021-08-11T16:20:03.272887Z" + "end_time": "2021-10-11T10:53:25.527624Z", + "start_time": "2021-10-11T10:53:25.518458Z" } - } - }, - { - "cell_type": "code", - "execution_count": 7, - "source": [ - "model.c_out" - ], + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ - "1000" + "MC constructor\n", + " c_in: 3, c_out: 1000\n", + " expansion: 1, groups: 1, dw: False, div_groups: None\n", + " sa: False, se: False\n", + " stem sizes: [3, 32, 32, 64], stride on 0\n", + " body sizes [64, 128, 256, 512]\n", + " layers: [2, 2, 2, 2]" ] }, + "execution_count": 3, "metadata": {}, - "execution_count": 7 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:03.461835Z", - "start_time": "2021-08-11T16:20:03.456422Z" - } - } - }, - { - "cell_type": "code", - "execution_count": 8, "source": [ - "model.stem_sizes" - ], - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "[3, 32, 32, 64]" - ] - }, - "metadata": {}, - "execution_count": 8 - } - ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:03.674906Z", - "start_time": "2021-08-11T16:20:03.666383Z" - } - } + "mc" + ] }, { - "cell_type": "code", - "execution_count": 9, + "cell_type": "markdown", + "metadata": {}, "source": [ - "model.layers" - ], - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "[2, 2, 2, 2]" - ] - }, - "metadata": {}, - "execution_count": 9 - } - ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:03.870746Z", - "start_time": "2021-08-11T16:20:03.865386Z" - } - } + "Now we have model consructor, default setting as xresnet18. And we can get model after call it." + ] }, { "cell_type": "code", - "execution_count": 10, - "source": [ - "model.expansion" - ], - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "1" - ] - }, - "metadata": {}, - "execution_count": 10 - } - ], + "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:04.092045Z", - "start_time": "2021-08-11T16:20:04.088801Z" + "end_time": "2021-10-11T10:54:00.173159Z", + "start_time": "2021-10-11T10:53:59.977116Z" } - } - }, - { - "cell_type": "code", - "execution_count": 11, - "source": [ - "#collapse_output\n", - "model()" - ], + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", - " model Net\n", + " MC\n", " (stem): Sequential(\n", " (conv_0): ConvLayer(\n", " (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", @@ -484,57 +360,58 @@ ")" ] }, + "execution_count": 4, "metadata": {}, - "execution_count": 11 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:04.475422Z", - "start_time": "2021-08-11T16:20:04.277032Z" - } - } + "source": [ + "#collapse_output\n", + "model = mc()\n", + "model" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "If you want to change model, just change constructor parameters. \n", "Lets create xresnet50." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 12, - "source": [ - "model.expansion = 4\n", - "model.layers = [3,4,6,3]" - ], - "outputs": [], "metadata": { "ExecuteTime": { "end_time": "2021-08-11T16:20:04.731830Z", "start_time": "2021-08-11T16:20:04.725686Z" } - } + }, + "outputs": [], + "source": [ + "mc.expansion = 4\n", + "mc.layers = [3,4,6,3]" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Now we can look at model body and if we call constructor - we have pytorch model!" - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 13, - "source": [ - "#collapse_output\n", - "model.body" - ], + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T10:55:53.799327Z", + "start_time": "2021-10-11T10:55:53.722731Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", @@ -542,60 +419,27 @@ " (bl_0): ResBlock(\n", " (convs): Sequential(\n", " (conv_0): ConvLayer(\n", - " (conv): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): ReLU(inplace=True)\n", " )\n", " (conv_1): ConvLayer(\n", " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " )\n", - " (idconv): ConvLayer(\n", - " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", " (act_fn): ReLU(inplace=True)\n", " )\n", " (bl_1): ResBlock(\n", " (convs): Sequential(\n", " (conv_0): ConvLayer(\n", - " (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_1): ConvLayer(\n", " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): ReLU(inplace=True)\n", " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (bl_2): ResBlock(\n", - " (convs): Sequential(\n", - " (conv_0): ConvLayer(\n", - " (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", " (conv_1): ConvLayer(\n", " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " )\n", " (act_fn): ReLU(inplace=True)\n", @@ -605,80 +449,32 @@ " (bl_0): ResBlock(\n", " (convs): Sequential(\n", " (conv_0): ConvLayer(\n", - " (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (conv): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): ReLU(inplace=True)\n", " )\n", " (conv_1): ConvLayer(\n", - " (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", + " (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " )\n", " (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", " (idconv): ConvLayer(\n", - " (conv): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " (act_fn): ReLU(inplace=True)\n", " )\n", " (bl_1): ResBlock(\n", " (convs): Sequential(\n", " (conv_0): ConvLayer(\n", - " (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_1): ConvLayer(\n", - " (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (bl_2): ResBlock(\n", - " (convs): Sequential(\n", - " (conv_0): ConvLayer(\n", - " (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_1): ConvLayer(\n", " (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): ReLU(inplace=True)\n", " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (bl_3): ResBlock(\n", - " (convs): Sequential(\n", - " (conv_0): ConvLayer(\n", - " (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", " (conv_1): ConvLayer(\n", " (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " )\n", " (act_fn): ReLU(inplace=True)\n", @@ -688,118 +484,32 @@ " (bl_0): ResBlock(\n", " (convs): Sequential(\n", " (conv_0): ConvLayer(\n", - " (conv): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (conv): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): ReLU(inplace=True)\n", " )\n", " (conv_1): ConvLayer(\n", - " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " )\n", " (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", " (idconv): ConvLayer(\n", - " (conv): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " (act_fn): ReLU(inplace=True)\n", " )\n", " (bl_1): ResBlock(\n", " (convs): Sequential(\n", " (conv_0): ConvLayer(\n", - " (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_1): ConvLayer(\n", - " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (bl_2): ResBlock(\n", - " (convs): Sequential(\n", - " (conv_0): ConvLayer(\n", - " (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_1): ConvLayer(\n", " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): ReLU(inplace=True)\n", " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (bl_3): ResBlock(\n", - " (convs): Sequential(\n", - " (conv_0): ConvLayer(\n", - " (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_1): ConvLayer(\n", - " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (bl_4): ResBlock(\n", - " (convs): Sequential(\n", - " (conv_0): ConvLayer(\n", - " (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_1): ConvLayer(\n", - " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (bl_5): ResBlock(\n", - " (convs): Sequential(\n", - " (conv_0): ConvLayer(\n", - " (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", " (conv_1): ConvLayer(\n", " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " )\n", " (act_fn): ReLU(inplace=True)\n", @@ -809,61 +519,32 @@ " (bl_0): ResBlock(\n", " (convs): Sequential(\n", " (conv_0): ConvLayer(\n", - " (conv): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (conv): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): ReLU(inplace=True)\n", " )\n", " (conv_1): ConvLayer(\n", - " (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", + " (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " )\n", " (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n", " (idconv): ConvLayer(\n", - " (conv): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (conv): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " (act_fn): ReLU(inplace=True)\n", " )\n", " (bl_1): ResBlock(\n", " (convs): Sequential(\n", " (conv_0): ConvLayer(\n", - " (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_1): ConvLayer(\n", " (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): ReLU(inplace=True)\n", " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (bl_2): ResBlock(\n", - " (convs): Sequential(\n", - " (conv_0): ConvLayer(\n", - " (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", " (conv_1): ConvLayer(\n", " (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (act_fn): ReLU(inplace=True)\n", - " )\n", - " (conv_2): ConvLayer(\n", - " (conv): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " )\n", " )\n", " (act_fn): ReLU(inplace=True)\n", @@ -872,219 +553,220 @@ ")" ] }, + "execution_count": 5, "metadata": {}, - "execution_count": 13 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:06.826936Z", - "start_time": "2021-08-11T16:20:06.619963Z" - } - } + "source": [ + "#collapse_output\n", + "mc.body" + ] }, { "cell_type": "code", - "execution_count": 14, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 3, 128, 128)\n", - "y = model()(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 1000]), f\"size expected {bs_test}, 1000\"" - ], + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T10:56:22.059863Z", + "start_time": "2021-10-11T10:56:21.683019Z" + } + }, "outputs": [ { + "name": "stdout", "output_type": "stream", - "name": "stderr", "text": [ - "/home/jzz/anaconda3/envs/pt_19/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1623448278899/work/c10/core/TensorImpl.h:1156.)\n", - " return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n" + "torch.Size([16, 1000])\n" ] }, { + "name": "stderr", "output_type": "stream", - "name": "stdout", "text": [ - "torch.Size([16, 1000])\n" + "/home/jzz/anaconda3/envs/pt_19/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1623448278899/work/c10/core/TensorImpl.h:1156.)\n", + " return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n" ] } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:08.702628Z", - "start_time": "2021-08-11T16:20:07.651169Z" - } - } + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 3, 128, 128)\n", + "y = mc()(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 1000]), f\"size expected {bs_test}, 1000\"" + ] }, { "cell_type": "code", - "execution_count": 15, - "source": [ - "#hide\n", - "model.block_sizes" - ], + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T10:56:33.743384Z", + "start_time": "2021-10-11T10:56:33.739200Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ - "[16, 64, 128, 256, 512]" + "[64, 64, 128, 256, 512]" ] }, + "execution_count": 9, "metadata": {}, - "execution_count": 15 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:09.065294Z", - "start_time": "2021-08-11T16:20:09.053356Z" - } - } + "source": [ + "#hide\n", + "mc.block_sizes" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## More modification." - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Main purpose of this module - fast and easy modify model.\n", "And here is the link to more modification to beat Imagenette leaderboard with add MaxBlurPool and modification to ResBlock https://github.com/ayasyrev/imagenette_experiments/blob/master/ResnetTrick_create_model_fit.ipynb \n", "\n", "But now lets create model as mxresnet50 from fastai forums tread https://forums.fast.ai/t/how-we-beat-the-5-epoch-imagewoof-leaderboard-score-some-new-techniques-to-consider \n" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Lets create mxresnet constructor." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 16, - "source": [ - "model = Net(name='MxResNet')" - ], - "outputs": [], + "execution_count": 10, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:10.167851Z", - "start_time": "2021-08-11T16:20:10.161915Z" + "end_time": "2021-10-11T10:57:14.671755Z", + "start_time": "2021-10-11T10:57:14.668147Z" } - } + }, + "outputs": [], + "source": [ + "mc = ModelConstructor(name='MxResNet')" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Then lets modify stem." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": 17, - "source": [ - "model.stem_sizes = [3,32,64,64]" - ], - "outputs": [], "metadata": { "ExecuteTime": { "end_time": "2021-08-11T16:20:11.457736Z", "start_time": "2021-08-11T16:20:11.451735Z" } - } + }, + "outputs": [], + "source": [ + "mc.stem_sizes = [3,32,64,64]" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Now lets change activation function to Mish.\n", "Here is link to forum disscussion https://forums.fast.ai/t/meet-mish-new-activation-function-possible-successor-to-relu \n", - "Mish is in model_constructor.activations" - ], - "metadata": {} + "We'v got Mish is in model_constructor.activations, but from pytorch 1.9 take it from torch:" + ] }, { "cell_type": "code", - "execution_count": 18, - "source": [ - "from model_constructor.activations import Mish" - ], - "outputs": [], + "execution_count": 12, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:12.023094Z", - "start_time": "2021-08-11T16:20:12.002583Z" + "end_time": "2021-10-11T10:59:05.225156Z", + "start_time": "2021-10-11T10:59:05.220889Z" } - } + }, + "outputs": [], + "source": [ + "# from model_constructor.activations import Mish\n", + "from torch.nn import Mish" + ] }, { "cell_type": "code", - "execution_count": 19, - "source": [ - "model.act_fn = Mish()" - ], - "outputs": [], + "execution_count": 13, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:12.593047Z", - "start_time": "2021-08-11T16:20:12.588971Z" + "end_time": "2021-10-11T10:59:11.880575Z", + "start_time": "2021-10-11T10:59:11.877443Z" } - } + }, + "outputs": [], + "source": [ + "mc.act_fn = Mish()" + ] }, { "cell_type": "code", - "execution_count": 20, - "source": [ - "model" - ], + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T10:59:17.458923Z", + "start_time": "2021-10-11T10:59:17.454864Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "MxResNet constructor\n", " c_in: 3, c_out: 1000\n", - " expansion: 1, groups: 1, dw: False\n", + " expansion: 1, groups: 1, dw: False, div_groups: None\n", " sa: False, se: False\n", - " stem sizes: [3, 32, 64, 64], stide on 0\n", + " stem sizes: [3, 32, 32, 64], stride on 0\n", " body sizes [64, 128, 256, 512]\n", " layers: [2, 2, 2, 2]" ] }, + "execution_count": 15, "metadata": {}, - "execution_count": 20 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:13.852977Z", - "start_time": "2021-08-11T16:20:13.844538Z" - } - } + "source": [ + "mc" + ] }, { "cell_type": "code", - "execution_count": 21, - "source": [ - "#collapse_output\n", - "model()" - ], + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T10:59:29.960894Z", + "start_time": "2021-10-11T10:59:29.831906Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "Sequential(\n", - " model MxResNet\n", + " MxResNet\n", " (stem): Sequential(\n", " (conv_0): ConvLayer(\n", " (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", @@ -1092,12 +774,12 @@ " (act_fn): Mish()\n", " )\n", " (conv_1): ConvLayer(\n", - " (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): Mish()\n", " )\n", " (conv_2): ConvLayer(\n", - " (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): Mish()\n", " )\n", @@ -1248,127 +930,127 @@ ")" ] }, + "execution_count": 16, "metadata": {}, - "execution_count": 21 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:14.408942Z", - "start_time": "2021-08-11T16:20:14.239570Z" - } - } + "source": [ + "#collapse_output\n", + "mc()" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## MXResNet" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Now lets make MxResNet50" - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 22, - "source": [ - "model.expansion = 4\n", - "model.layers = [3,4,6,3]\n", - "model.name = 'mxresnet50'" - ], - "outputs": [], + "execution_count": 17, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:14.995845Z", - "start_time": "2021-08-11T16:20:14.989405Z" + "end_time": "2021-10-11T11:00:25.586474Z", + "start_time": "2021-10-11T11:00:25.583231Z" } - } + }, + "outputs": [], + "source": [ + "mc.expansion = 4\n", + "mc.layers = [3,4,6,3]\n", + "mc.name = 'mxresnet50'" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Now we have mxresnet50 constructor. \n", "We can inspect every parts of it. \n", "And after call it we got model." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 23, - "source": [ - "model" - ], + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:00:29.630053Z", + "start_time": "2021-10-11T11:00:29.621579Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "mxresnet50 constructor\n", " c_in: 3, c_out: 1000\n", - " expansion: 4, groups: 1, dw: False\n", + " expansion: 4, groups: 1, dw: False, div_groups: None\n", " sa: False, se: False\n", - " stem sizes: [3, 32, 64, 64], stide on 0\n", + " stem sizes: [3, 32, 32, 64], stride on 0\n", " body sizes [64, 128, 256, 512]\n", " layers: [3, 4, 6, 3]" ] }, + "execution_count": 18, "metadata": {}, - "execution_count": 23 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:15.577567Z", - "start_time": "2021-08-11T16:20:15.568883Z" - } - } + "source": [ + "mc" + ] }, { "cell_type": "code", - "execution_count": 24, - "source": [ - "#collapse_output\n", - "model.stem.conv_1" - ], + "execution_count": 19, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:00:39.078112Z", + "start_time": "2021-10-11T11:00:39.072528Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "ConvLayer(\n", - " (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", + " (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", + " (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (act_fn): Mish()\n", ")" ] }, + "execution_count": 19, "metadata": {}, - "execution_count": 24 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:15.827743Z", - "start_time": "2021-08-11T16:20:15.821164Z" - } - } + "source": [ + "#collapse_output\n", + "mc.stem.conv_1" + ] }, { "cell_type": "code", - "execution_count": 25, - "source": [ - "#collapse_output\n", - "model.body.l_0.bl_0" - ], + "execution_count": 20, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:00:48.437461Z", + "start_time": "2021-10-11T11:00:48.280337Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "ResBlock(\n", @@ -1396,143 +1078,143 @@ ")" ] }, + "execution_count": 20, "metadata": {}, - "execution_count": 25 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:16.233030Z", - "start_time": "2021-08-11T16:20:16.070654Z" - } - } + "source": [ + "#collapse_output\n", + "mc.body.l_0.bl_0" + ] }, { "cell_type": "code", - "execution_count": 26, - "source": [ - "#hide\n", - "bs_test = 16\n", - "xb = torch.randn(bs_test, 3, 128, 128)\n", - "y = model()(xb)\n", - "print(y.shape)\n", - "assert y.shape == torch.Size([bs_test, 1000]), f\"size expected {bs_test}, 1000\"" - ], + "execution_count": 21, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:00:59.366012Z", + "start_time": "2021-10-11T11:00:58.231504Z" + } + }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 1000])\n" ] } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:17.605597Z", - "start_time": "2021-08-11T16:20:16.311621Z" - } - } + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 3, 128, 128)\n", + "y = mc()(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 1000]), f\"size expected {bs_test}, 1000\"" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## YaResNet" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Now lets change Resblock to YaResBlock (Yet another ResNet, former NewResBlock) is in lib from version 0.1.0" - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 27, - "source": [ - "from model_constructor.yaresnet import YaResBlock" - ], - "outputs": [], + "execution_count": 22, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:18.917006Z", - "start_time": "2021-08-11T16:20:18.913531Z" + "end_time": "2021-10-11T11:01:26.539296Z", + "start_time": "2021-10-11T11:01:26.529382Z" } - } + }, + "outputs": [], + "source": [ + "from model_constructor.yaresnet import YaResBlock" + ] }, { "cell_type": "code", - "execution_count": 28, - "source": [ - "model.block = YaResBlock" - ], - "outputs": [], + "execution_count": 23, "metadata": { "ExecuteTime": { - "end_time": "2021-08-11T16:20:19.631481Z", - "start_time": "2021-08-11T16:20:19.625715Z" + "end_time": "2021-10-11T11:01:33.276800Z", + "start_time": "2021-10-11T11:01:33.271400Z" } - } + }, + "outputs": [], + "source": [ + "mc.block = YaResBlock" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "That all. Now we have YaResNet constructor" - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 29, - "source": [ - "#collapse_output\n", - "model.name = 'YaResNet'\n", - "model" - ], + "execution_count": 24, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:01:41.003717Z", + "start_time": "2021-10-11T11:01:40.999816Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "YaResNet constructor\n", " c_in: 3, c_out: 1000\n", - " expansion: 4, groups: 1, dw: False\n", + " expansion: 4, groups: 1, dw: False, div_groups: None\n", " sa: False, se: False\n", - " stem sizes: [3, 32, 64, 64], stide on 0\n", + " stem sizes: [3, 32, 32, 64], stride on 0\n", " body sizes [64, 128, 256, 512]\n", " layers: [3, 4, 6, 3]" ] }, + "execution_count": 24, "metadata": {}, - "execution_count": 29 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:20.316973Z", - "start_time": "2021-08-11T16:20:20.307829Z" - } - } + "source": [ + "#collapse_output\n", + "mc.name = 'YaResNet'\n", + "mc" + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Let see what we have." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 30, - "source": [ - "#collapse_output\n", - "model.body.l_1.bl_0" - ], + "execution_count": 25, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:01:49.871498Z", + "start_time": "2021-10-11T11:01:49.727231Z" + } + }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "YaResBlock(\n", @@ -1561,43 +1243,143 @@ ")" ] }, + "execution_count": 25, "metadata": {}, - "execution_count": 30 + "output_type": "execute_result" } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:20.957456Z", - "start_time": "2021-08-11T16:20:20.803032Z" - } - } + "source": [ + "#collapse_output\n", + "mc.body.l_1.bl_0" + ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 26, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:01:58.583051Z", + "start_time": "2021-10-11T11:01:57.521723Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([16, 1000])\n" + ] + } + ], "source": [ "#hide\n", "bs_test = 16\n", "xb = torch.randn(bs_test, 3, 128, 128)\n", - "y = model()(xb)\n", + "y = mc()(xb)\n", "print(y.shape)\n", "assert y.shape == torch.Size([bs_test, 1000]), f\"size expected {bs_test}, 1000\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## First version" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First version, it deprecated, but still here for compatability." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:03:30.241159Z", + "start_time": "2021-10-11T11:03:30.235513Z" + } + }, + "outputs": [], + "source": [ + "from model_constructor.net import Net" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:03:36.551041Z", + "start_time": "2021-10-11T11:03:36.546458Z" + } + }, + "outputs": [], + "source": [ + "mc = Net()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:03:39.559796Z", + "start_time": "2021-10-11T11:03:39.555460Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Net constructor\n", + " c_in: 3, c_out: 1000\n", + " expansion: 1, groups: 1, dw: False, div_groups: None\n", + " sa: False, se: False\n", + " stem sizes: [3, 32, 32, 64], stride on 0\n", + " body sizes [64, 128, 256, 512]\n", + " layers: [2, 2, 2, 2]" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } ], + "source": [ + "mc" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-11T11:04:20.417154Z", + "start_time": "2021-10-11T11:04:20.067581Z" + } + }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "torch.Size([16, 1000])\n" ] } ], - "metadata": { - "ExecuteTime": { - "end_time": "2021-08-11T16:20:22.223523Z", - "start_time": "2021-08-11T16:20:21.037135Z" - } - } + "source": [ + "#hide\n", + "bs_test = 16\n", + "xb = torch.randn(bs_test, 3, 128, 128)\n", + "y = mc()(xb)\n", + "print(y.shape)\n", + "assert y.shape == torch.Size([bs_test, 1000]), f\"size expected {bs_test}, 1000\"" + ] } ], "metadata": { @@ -1634,4 +1416,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/README.md b/README.md index e2890e5..53fd744 100644 --- a/README.md +++ b/README.md @@ -12,30 +12,30 @@ Or install from repo: ## How to use -First import constructor class, then create model constructor oject. +First import constructor class, then create model constructor object. Now you can change every part of model. ```python -from model_constructor.net import Net +from model_constructor import ModelConstructor ``` ```python -model = Net() +mc = ModelConstructor() ``` ```python -model +mc ``` - Net constructor + MC constructor c_in: 3, c_out: 1000 - expansion: 1, groups: 1, dw: False + expansion: 1, groups: 1, dw: False, div_groups: None sa: False, se: False - stem sizes: [3, 32, 32, 64], stide on 0 + stem sizes: [3, 32, 32, 64], stride on 0 body sizes [64, 128, 256, 512] layers: [2, 2, 2, 2] @@ -43,64 +43,10 @@ model Now we have model consructor, default setting as xresnet18. And we can get model after call it. -```python -model.c_in -``` - - - - - 3 - - - -```python -model.c_out -``` - - - - - 1000 - - - -```python -model.stem_sizes -``` - - - - - [3, 32, 32, 64] - - - -```python -model.layers -``` - - - - - [2, 2, 2, 2] - - - -```python -model.expansion -``` - - - - - 1 - - - ```python #collapse_output -model() +model = mc() +model ```
Output details ... @@ -109,7 +55,7 @@ model() Sequential( - model Net + MC (stem): Sequential( (conv_0): ConvLayer( (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) @@ -280,15 +226,15 @@ If you want to change model, just change constructor parameters. Lets create xresnet50. ```python -model.expansion = 4 -model.layers = [3,4,6,3] +mc.expansion = 4 +mc.layers = [3,4,6,3] ``` Now we can look at model body and if we call constructor - we have pytorch model! ```python #collapse_output -model.body +mc.body ```
Output details ... @@ -301,60 +247,27 @@ model.body (bl_0): ResBlock( (convs): Sequential( (conv_0): ConvLayer( - (conv): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) + (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): ReLU(inplace=True) ) (conv_1): ConvLayer( (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) - (idconv): ConvLayer( - (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) (act_fn): ReLU(inplace=True) ) (bl_1): ResBlock( (convs): Sequential( (conv_0): ConvLayer( - (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_1): ConvLayer( (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): ReLU(inplace=True) ) - (conv_2): ConvLayer( - (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - ) - (act_fn): ReLU(inplace=True) - ) - (bl_2): ResBlock( - (convs): Sequential( - (conv_0): ConvLayer( - (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) (conv_1): ConvLayer( (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (act_fn): ReLU(inplace=True) @@ -364,80 +277,32 @@ model.body (bl_0): ResBlock( (convs): Sequential( (conv_0): ConvLayer( - (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (conv): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): ReLU(inplace=True) ) (conv_1): ConvLayer( - (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (pool): AvgPool2d(kernel_size=2, stride=2, padding=0) (idconv): ConvLayer( - (conv): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) (act_fn): ReLU(inplace=True) ) (bl_1): ResBlock( (convs): Sequential( (conv_0): ConvLayer( - (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_1): ConvLayer( - (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) - (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - ) - (act_fn): ReLU(inplace=True) - ) - (bl_2): ResBlock( - (convs): Sequential( - (conv_0): ConvLayer( - (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_1): ConvLayer( (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): ReLU(inplace=True) ) - (conv_2): ConvLayer( - (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - ) - (act_fn): ReLU(inplace=True) - ) - (bl_3): ResBlock( - (convs): Sequential( - (conv_0): ConvLayer( - (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) (conv_1): ConvLayer( (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (act_fn): ReLU(inplace=True) @@ -447,118 +312,32 @@ model.body (bl_0): ResBlock( (convs): Sequential( (conv_0): ConvLayer( - (conv): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (conv): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): ReLU(inplace=True) ) (conv_1): ConvLayer( - (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (pool): AvgPool2d(kernel_size=2, stride=2, padding=0) (idconv): ConvLayer( - (conv): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) (act_fn): ReLU(inplace=True) ) (bl_1): ResBlock( (convs): Sequential( (conv_0): ConvLayer( - (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_1): ConvLayer( - (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - ) - (act_fn): ReLU(inplace=True) - ) - (bl_2): ResBlock( - (convs): Sequential( - (conv_0): ConvLayer( - (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_1): ConvLayer( - (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - ) - (act_fn): ReLU(inplace=True) - ) - (bl_3): ResBlock( - (convs): Sequential( - (conv_0): ConvLayer( - (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_1): ConvLayer( - (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - ) - (act_fn): ReLU(inplace=True) - ) - (bl_4): ResBlock( - (convs): Sequential( - (conv_0): ConvLayer( - (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_1): ConvLayer( (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): ReLU(inplace=True) ) - (conv_2): ConvLayer( - (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - ) - (act_fn): ReLU(inplace=True) - ) - (bl_5): ResBlock( - (convs): Sequential( - (conv_0): ConvLayer( - (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) (conv_1): ConvLayer( (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (act_fn): ReLU(inplace=True) @@ -568,61 +347,32 @@ model.body (bl_0): ResBlock( (convs): Sequential( (conv_0): ConvLayer( - (conv): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (conv): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): ReLU(inplace=True) ) (conv_1): ConvLayer( - (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (pool): AvgPool2d(kernel_size=2, stride=2, padding=0) (idconv): ConvLayer( - (conv): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) + (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) (act_fn): ReLU(inplace=True) ) (bl_1): ResBlock( (convs): Sequential( (conv_0): ConvLayer( - (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_1): ConvLayer( (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): ReLU(inplace=True) ) - (conv_2): ConvLayer( - (conv): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - ) - (act_fn): ReLU(inplace=True) - ) - (bl_2): ResBlock( - (convs): Sequential( - (conv_0): ConvLayer( - (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) (conv_1): ConvLayer( (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - (act_fn): ReLU(inplace=True) - ) - (conv_2): ConvLayer( - (conv): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False) - (bn): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) ) (act_fn): ReLU(inplace=True) @@ -645,29 +395,30 @@ But now lets create model as mxresnet50 from fastai forums tread https://forums. Lets create mxresnet constructor. ```python -model = Net(name='MxResNet') +mc = ModelConstructor(name='MxResNet') ``` Then lets modify stem. ```python -model.stem_sizes = [3,32,64,64] +mc.stem_sizes = [3,32,64,64] ``` Now lets change activation function to Mish. Here is link to forum disscussion https://forums.fast.ai/t/meet-mish-new-activation-function-possible-successor-to-relu -Mish is in model_constructor.activations +We'v got Mish is in model_constructor.activations, but from pytorch 1.9 take it from torch: ```python -from model_constructor.activations import Mish +# from model_constructor.activations import Mish +from torch.nn import Mish ``` ```python -model.act_fn = Mish() +mc.act_fn = Mish() ``` ```python -model +mc ``` @@ -675,9 +426,9 @@ model MxResNet constructor c_in: 3, c_out: 1000 - expansion: 1, groups: 1, dw: False + expansion: 1, groups: 1, dw: False, div_groups: None sa: False, se: False - stem sizes: [3, 32, 64, 64], stide on 0 + stem sizes: [3, 32, 32, 64], stride on 0 body sizes [64, 128, 256, 512] layers: [2, 2, 2, 2] @@ -685,7 +436,7 @@ model ```python #collapse_output -model() +mc() ```
Output details ... @@ -694,7 +445,7 @@ model() Sequential( - model MxResNet + MxResNet (stem): Sequential( (conv_0): ConvLayer( (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) @@ -702,12 +453,12 @@ model() (act_fn): Mish() ) (conv_1): ConvLayer( - (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) - (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): Mish() ) (conv_2): ConvLayer( - (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): Mish() ) @@ -866,9 +617,9 @@ model() Now lets make MxResNet50 ```python -model.expansion = 4 -model.layers = [3,4,6,3] -model.name = 'mxresnet50' +mc.expansion = 4 +mc.layers = [3,4,6,3] +mc.name = 'mxresnet50' ``` Now we have mxresnet50 constructor. @@ -876,7 +627,7 @@ We can inspect every parts of it. And after call it we got model. ```python -model +mc ``` @@ -884,9 +635,9 @@ model mxresnet50 constructor c_in: 3, c_out: 1000 - expansion: 4, groups: 1, dw: False + expansion: 4, groups: 1, dw: False, div_groups: None sa: False, se: False - stem sizes: [3, 32, 64, 64], stide on 0 + stem sizes: [3, 32, 32, 64], stride on 0 body sizes [64, 128, 256, 512] layers: [3, 4, 6, 3] @@ -894,7 +645,7 @@ model ```python #collapse_output -model.stem.conv_1 +mc.stem.conv_1 ```
Output details ... @@ -903,8 +654,8 @@ model.stem.conv_1 ConvLayer( - (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) - (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) (act_fn): Mish() ) @@ -914,7 +665,7 @@ model.stem.conv_1 ```python #collapse_output -model.body.l_0.bl_0 +mc.body.l_0.bl_0 ```
Output details ... @@ -959,15 +710,15 @@ from model_constructor.yaresnet import YaResBlock ``` ```python -model.block = YaResBlock +mc.block = YaResBlock ``` That all. Now we have YaResNet constructor ```python #collapse_output -model.name = 'YaResNet' -model +mc.name = 'YaResNet' +mc ```
Output details ... @@ -977,9 +728,9 @@ model YaResNet constructor c_in: 3, c_out: 1000 - expansion: 4, groups: 1, dw: False + expansion: 4, groups: 1, dw: False, div_groups: None sa: False, se: False - stem sizes: [3, 32, 64, 64], stide on 0 + stem sizes: [3, 32, 32, 64], stride on 0 body sizes [64, 128, 256, 512] layers: [3, 4, 6, 3] @@ -991,7 +742,7 @@ Let see what we have. ```python #collapse_output -model.body.l_1.bl_0 +mc.body.l_1.bl_0 ```
Output details ... @@ -1027,3 +778,32 @@ model.body.l_1.bl_0
+ +## First version + +First version, it deprecated, but still here for compatability. + +```python +from model_constructor.net import Net +``` + +```python +mc = Net() +``` + +```python +mc +``` + + + + + Net constructor + c_in: 3, c_out: 1000 + expansion: 1, groups: 1, dw: False, div_groups: None + sa: False, se: False + stem sizes: [3, 32, 32, 64], stride on 0 + body sizes [64, 128, 256, 512] + layers: [2, 2, 2, 2] + + diff --git a/docs/ModelConstructor.html b/docs/ModelConstructor.html new file mode 100644 index 0000000..d725f84 --- /dev/null +++ b/docs/ModelConstructor.html @@ -0,0 +1,961 @@ +--- + +title: Model constructor. + + +keywords: fastai +sidebar: home_sidebar + +summary: "Create and tune pytorch model." +description: "Create and tune pytorch model." +nb_path: "Nbs/00_ModelConstructor.ipynb" +--- + + +
+ + {% raw %} + +
+ +
+ {% endraw %} + +
+
+

ResBlock

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

class ResBlock[source]

ResBlock(expansion, ni, nh, stride=1, conv_layer=ConvLayer, act_fn=ReLU(inplace=True), zero_bn=True, bn_1st=True, pool=AvgPool2d(kernel_size=2, stride=2, padding=0), sa=False, sym=False, groups=1, dw=False, div_groups=None, se_module=SEModule, se=False, se_reduction=16) :: Module

+
+

Resnet block

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
block = ResBlock(1,64,64,sa=True)
+block
+
+ +
+
+
+ +
+
+ +
+ + + +
+
ResBlock(
+  (convs): Sequential(
+    (conv_0): ConvLayer(
+      (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      (act_fn): ReLU(inplace=True)
+    )
+    (conv_1): ConvLayer(
+      (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    )
+    (sa): SimpleSelfAttention(
+      (conv): Conv1d(64, 64, kernel_size=(1,), stride=(1,), bias=False)
+    )
+  )
+  (act_fn): ReLU(inplace=True)
+)
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
block = ResBlock(4,64,64,sa=True, dw=True)
+block
+
+ +
+
+
+ +
+
+ +
+ + + +
+
ResBlock(
+  (convs): Sequential(
+    (conv_0): ConvLayer(
+      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      (act_fn): ReLU(inplace=True)
+    )
+    (conv_1): ConvLayer(
+      (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64, bias=False)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      (act_fn): ReLU(inplace=True)
+    )
+    (conv_2): ConvLayer(
+      (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
+      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    )
+    (sa): SimpleSelfAttention(
+      (conv): Conv1d(256, 256, kernel_size=(1,), stride=(1,), bias=False)
+    )
+  )
+  (act_fn): ReLU(inplace=True)
+)
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
block = ResBlock(4,64,64,sa=True, groups=4)
+block
+
+ +
+
+
+ +
+
+ +
+ + + +
+
ResBlock(
+  (convs): Sequential(
+    (conv_0): ConvLayer(
+      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      (act_fn): ReLU(inplace=True)
+    )
+    (conv_1): ConvLayer(
+      (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=4, bias=False)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      (act_fn): ReLU(inplace=True)
+    )
+    (conv_2): ConvLayer(
+      (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
+      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    )
+    (sa): SimpleSelfAttention(
+      (conv): Conv1d(256, 256, kernel_size=(1,), stride=(1,), bias=False)
+    )
+  )
+  (act_fn): ReLU(inplace=True)
+)
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
block = ResBlock(2,64,64,act_fn=nn.LeakyReLU(), bn_1st=False)
+block
+
+ +
+
+
+ +
+
+ +
+ + + +
+
ResBlock(
+  (convs): Sequential(
+    (conv_0): ConvLayer(
+      (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
+      (act_fn): LeakyReLU(negative_slope=0.01)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    )
+    (conv_1): ConvLayer(
+      (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+      (act_fn): LeakyReLU(negative_slope=0.01)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    )
+    (conv_2): ConvLayer(
+      (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
+      (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    )
+  )
+  (act_fn): LeakyReLU(negative_slope=0.01)
+)
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
block = ResBlock(2, 64, 64, sa=True, se=True)
+block
+
+ +
+
+
+ +
+
+ +
+ + + +
+
ResBlock(
+  (convs): Sequential(
+    (conv_0): ConvLayer(
+      (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      (act_fn): ReLU(inplace=True)
+    )
+    (conv_1): ConvLayer(
+      (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      (act_fn): ReLU(inplace=True)
+    )
+    (conv_2): ConvLayer(
+      (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
+      (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    )
+    (se): SEModule(
+      (squeeze): AdaptiveAvgPool2d(output_size=1)
+      (excitation): Sequential(
+        (fc_reduce): Linear(in_features=128, out_features=8, bias=True)
+        (se_act): ReLU(inplace=True)
+        (fc_expand): Linear(in_features=8, out_features=128, bias=True)
+        (se_gate): Sigmoid()
+      )
+    )
+    (sa): SimpleSelfAttention(
+      (conv): Conv1d(128, 128, kernel_size=(1,), stride=(1,), bias=False)
+    )
+  )
+  (act_fn): ReLU(inplace=True)
+)
+
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

Stem, Body, Head

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

_make_layer[source]

_make_layer(expansion, ni, nf, blocks, stride, sa)

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

_make_stem[source]

_make_stem()

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

_make_body[source]

_make_body()

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

_make_head[source]

_make_head()

+
+ +
+ +
+ +
+
+ +
+ {% endraw %} + +
+
+

Model Constructor.

+
+
+
+ {% raw %} + +
+ +
+
+ +
+ + +
+

class ModelConstructor[source]

ModelConstructor(name='MC', c_in=3, c_out=1000, block=ResBlock, conv_layer=ConvLayer, block_sizes=[64, 128, 256, 512], layers=[2, 2, 2, 2], norm=BatchNorm2d, act_fn=ReLU(inplace=True), pool=AvgPool2d(kernel_size=2, stride=2, padding=0), expansion=1, groups=1, dw=False, div_groups=None, sa=False, se=False, se_module=SEModule, se_reduction=16, bn_1st=True, zero_bn=True, stem_stride_on=0, stem_sizes=[32, 32, 64], stem_pool=MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False), stem_bn_end=False, _init_cnn=init_cnn, _make_stem=_make_stem, _make_layer=_make_layer, _make_body=_make_body, _make_head=_make_head)

+
+

Model constructor. As default - xresnet18

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc  = ModelConstructor()
+mc
+
+ +
+
+
+ +
+
+ +
+ + + +
+
MC constructor
+  c_in: 3, c_out: 1000
+  expansion: 1, groups: 1, dw: False, div_groups: None
+  sa: False, se: False
+  stem sizes: [3, 32, 32, 64], stride on 0
+  body sizes [64, 128, 256, 512]
+  layers: [2, 2, 2, 2]
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc._block_sizes
+
+ +
+
+
+ +
+
+ +
+ + + +
+
[64, 128, 256, 512]
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc.block_sizes
+
+ +
+
+
+ +
+
+ +
+ + + +
+
[64, 64, 128, 256, 512]
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc._block_sizes = [128, 256, 512, 1024]
+mc
+
+ +
+
+
+ +
+
+ +
+ + + +
+
MC constructor
+  c_in: 3, c_out: 1000
+  expansion: 1, groups: 1, dw: False, div_groups: None
+  sa: False, se: False
+  stem sizes: [3, 32, 32, 64], stride on 0
+  body sizes [128, 256, 512, 1024]
+  layers: [2, 2, 2, 2]
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc.block_sizes
+
+ +
+
+
+ +
+
+ +
+ + + +
+
[64, 128, 256, 512, 1024]
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc.block_sizes
+
+ +
+
+
+ +
+
+ +
+ + + +
+
[64, 64, 128, 256, 512]
+
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
model = ModelConstructor()
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc.stem
+
+ +
+
+
+
+ + + +
+
+ +
+ + + +
+
Sequential(
+  (conv_0): ConvLayer(
+    (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+    (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    (act_fn): ReLU(inplace=True)
+  )
+  (conv_1): ConvLayer(
+    (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+    (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    (act_fn): ReLU(inplace=True)
+  )
+  (conv_2): ConvLayer(
+    (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+    (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    (act_fn): ReLU(inplace=True)
+  )
+  (stem_pool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
+)
+
+ +
+ +
+
+ +
+
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc.stem_stride_on = 1
+mc.stem
+
+ +
+
+
+
+ + + +
+
+ +
+ + + +
+
Sequential(
+  (conv_0): ConvLayer(
+    (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+    (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    (act_fn): ReLU(inplace=True)
+  )
+  (conv_1): ConvLayer(
+    (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+    (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    (act_fn): ReLU(inplace=True)
+  )
+  (conv_2): ConvLayer(
+    (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+    (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    (act_fn): ReLU(inplace=True)
+  )
+  (stem_pool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
+)
+
+ +
+ +
+
+ +
+
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc.bn_1st = False
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc.act_fn =nn.LeakyReLU(inplace=True)
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc.sa = True
+mc.se = True
+
+ +
+
+
+ +
+ {% endraw %} + + {% raw %} + +
+
+ +
+
+
mc.body.l_1
+
+ +
+
+
+
+ + + +
+
+ +
+ + + +
+
Sequential(
+  (bl_0): ResBlock(
+    (convs): Sequential(
+      (conv_0): ConvLayer(
+        (conv): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+        (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)
+        (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      )
+      (conv_1): ConvLayer(
+        (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+        (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      )
+      (se): SEModule(
+        (squeeze): AdaptiveAvgPool2d(output_size=1)
+        (excitation): Sequential(
+          (fc_reduce): Linear(in_features=128, out_features=8, bias=True)
+          (se_act): ReLU(inplace=True)
+          (fc_expand): Linear(in_features=8, out_features=128, bias=True)
+          (se_gate): Sigmoid()
+        )
+      )
+    )
+    (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)
+    (idconv): ConvLayer(
+      (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
+      (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    )
+    (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)
+  )
+  (bl_1): ResBlock(
+    (convs): Sequential(
+      (conv_0): ConvLayer(
+        (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+        (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)
+        (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      )
+      (conv_1): ConvLayer(
+        (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+        (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+      )
+      (se): SEModule(
+        (squeeze): AdaptiveAvgPool2d(output_size=1)
+        (excitation): Sequential(
+          (fc_reduce): Linear(in_features=128, out_features=8, bias=True)
+          (se_act): ReLU(inplace=True)
+          (fc_expand): Linear(in_features=8, out_features=128, bias=True)
+          (se_gate): Sigmoid()
+        )
+      )
+    )
+    (act_fn): LeakyReLU(negative_slope=0.01, inplace=True)
+  )
+)
+
+ +
+ +
+
+ +
+
+ {% endraw %} + +
+
+

end

model_constructor +by ayasyrev

+ +
+
+
+
+ + diff --git a/docs/Net.html b/docs/Net.html index 4c62829..ee93995 100644 --- a/docs/Net.html +++ b/docs/Net.html @@ -1,21 +1,21 @@ --- -title: Model constructor class. +title: First version. Net. keywords: fastai sidebar: home_sidebar -summary: "Create and tune pytorch model." -description: "Create and tune pytorch model." -nb_path: "Nbs/00_Net.ipynb" +summary: "First version of constructor." +description: "First version of constructor." +nb_path: "Nbs/09_Net.ipynb" --- diff --git a/docs/YaResNet.html b/docs/YaResNet.html index c9446a9..0da6e81 100644 --- a/docs/YaResNet.html +++ b/docs/YaResNet.html @@ -37,6 +37,31 @@

YaResBlock {% raw %} +
+ +
+
+ +
+ + +
+

class YaResBlock[source]

YaResBlock(expansion, ni, nh, stride=1, conv_layer=ConvLayer, act_fn=ReLU(inplace=True), zero_bn=True, bn_1st=True, pool=AvgPool2d(kernel_size=2, stride=2, padding=0), sa=False, sym=False, groups=1, dw=False, div_groups=None, se_module=SEModule, se=False, se_reduction=16) :: Module

+
+

YaResBlock. Reduce by pool instead of stride 2

+ +
+ +
+ +
+
+ +
+ {% endraw %} + + {% raw %} +
@@ -126,13 +151,13 @@

YaResBlock YaResNet constructor.
-
yaresnet  = Net(block=YaResBlock, stem_sizes = [3, 32, 64, 64], name='YaResNet')
+
from model_constructor import ModelConstructor
+yaresnet  = ModelConstructor(block=YaResBlock, stem_sizes = [3, 32, 64, 64], name='YaResNet')
@@ -440,9 +466,9 @@

YaResNet constructor. 
YaResNet constructor
   c_in: 3, c_out: 1000
-  expansion: 1, groups: 1, dw: False
+  expansion: 1, groups: 1, dw: False, div_groups: None
   sa: False, se: False
-  stem sizes: [3, 32, 64, 64], stide on 0
+  stem sizes: [3, 32, 64, 64], stride on 0
   body sizes [64, 128, 256, 512]
   layers: [2, 2, 2, 2]

@@ -770,7 +796,7 @@

YaResNet constructor.
-
yaresnet.act_fn = Mish()
+
yaresnet.act_fn = torch.nn.Mish()
 yaresnet()
 

 
@@ -790,7 +816,7 @@ 
YaResNet constructor.
 
Sequential(
-  model YaResNet
+  YaResNet
   (stem): Sequential(
     (conv_0): ConvLayer(
       (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
@@ -963,120 +989,6 @@ 
YaResNet constructor.
     {% endraw %}
 
-

-

-
Predefined YResNet models. 

-

-

-

-    {% raw %}
-    
-

-

-
-

-    

-
yaresnet_parameters = {'block': YaResBlock, 'stem_sizes': [3, 32, 64, 64], 'act_fn': Mish(), 'stem_stride_on': 1}
-yaresnet34 = partial(Net, name='YaResnet34', expansion=1, layers=[3, 4, 6, 3], **yaresnet_parameters)
-yaresnet50 = partial(Net, name='YaResnet50', expansion=4, layers=[3, 4, 6, 3], **yaresnet_parameters)
-

-
-    

-

-

-
-

-    {% endraw %}
-
-    {% raw %}
-    
-

-

-
-

-    

-
model = yaresnet50(c_out=10)
-

-
-    

-

-

-
-

-    {% endraw %}
-
-    {% raw %}
-    
-

-

-
-

-    

-
model
-

-
-    

-

-

-
-

-

-
-

-
-
-
-

-
YaResnet50 constructor
-  c_in: 3, c_out: 10
-  expansion: 4, groups: 1, dw: False
-  sa: False, se: False
-  stem sizes: [3, 32, 64, 64], stide on 1
-  body sizes [64, 128, 256, 512]
-  layers: [3, 4, 6, 3]

-

-
-

-
-

-

-
-

-    {% endraw %}
-
-    {% raw %}
-    
-

-

-
-

-    

-
model.c_out, model.layers
-

-
-    

-

-

-
-

-

-
-

-
-
-
-

-
(10, [3, 4, 6, 3])

-

-
-

-
-

-

-
-

-    {% endraw %}
-
 

 

 
end 
model_constructor
diff --git a/docs/_data/sidebars/home_sidebar.yml b/docs/_data/sidebars/home_sidebar.yml
index 72d7345..ad2b994 100644
--- a/docs/_data/sidebars/home_sidebar.yml
+++ b/docs/_data/sidebars/home_sidebar.yml
@@ -10,8 +10,8 @@ entries:
       title: Overview
       url: /
     - output: web,pdf
-      title: Model constructor class.
-      url: Net.html
+      title: Model constructor.
+      url: ModelConstructor.html
     - output: web,pdf
       title: Activations functions.
       url: activations.html
@@ -30,6 +30,9 @@ entries:
     - output: web,pdf
       title: ConvMixer
       url: ConvMixer.html
+    - output: web,pdf
+      title: First version. Net.
+      url: Net.html
     - output: web,pdf
       title: Base constructor
       url: base_constructor.html
diff --git a/docs/activations.html b/docs/activations.html
index ce56e97..1aeba5c 100644
--- a/docs/activations.html
+++ b/docs/activations.html
@@ -43,6 +43,13 @@
 https://github.com/digantamisra98/Mish

 fastai forum discussion https://forums.fast.ai/t/meet-mish-new-activation-function-possible-successor-to-relu

 
+

+

+

+

+

+
Mish is in Pytorch from version 1.9. Use this version!

+
 

 

 

diff --git a/docs/index.html b/docs/index.html
index bf7aaab..4d6fe78 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -64,7 +64,7 @@ 
How to use 
 

 

-
First import constructor class, then create model constructor oject.

+
First import constructor class, then create model constructor object.

 
Now you can change every part of model.

 
 

@@ -77,7 +77,7 @@ 
How to use 
     

-
from model_constructor.net import Net
+
from model_constructor import ModelConstructor
 

 
     

@@ -94,7 +94,7 @@ 
How to use 
     

-
model = Net()
+
mc = ModelConstructor()
 

 
     

@@ -111,7 +111,7 @@ 
How to use 
     

-
model
+
mc
 

 
     

@@ -126,11 +126,11 @@ 
How to use 
-
Net constructor
+
MC constructor
   c_in: 3, c_out: 1000
-  expansion: 1, groups: 1, dw: False
+  expansion: 1, groups: 1, dw: False, div_groups: None
   sa: False, se: False
-  stem sizes: [3, 32, 32, 64], stide on 0
+  stem sizes: [3, 32, 32, 64], stride on 0
   body sizes [64, 128, 256, 512]
   layers: [2, 2, 2, 2]

 

@@ -157,172 +157,8 @@ 
How to use 
     

-
model.c_in
-

-
-    

-

-

-
-

-

-
-

-
-
-
-

-
3

-

-
-

-
-

-

-
-

-    {% endraw %}
-
-    {% raw %}
-    
-

-

-
-

-    

-
model.c_out
-

-
-    

-

-

-
-

-

-
-

-
-
-
-

-
1000

-

-
-

-
-

-

-
-

-    {% endraw %}
-
-    {% raw %}
-    
-

-

-
-

-    

-
model.stem_sizes
-

-
-    

-

-

-
-

-

-
-

-
-
-
-

-
[3, 32, 32, 64]

-

-
-

-
-

-

-
-

-    {% endraw %}
-
-    {% raw %}
-    
-

-

-
-

-    

-
model.layers
-

-
-    

-

-

-
-

-

-
-

-
-
-
-

-
[2, 2, 2, 2]

-

-
-

-
-

-

-
-

-    {% endraw %}
-
-    {% raw %}
-    
-

-

-
-

-    

-
model.expansion
-

-
-    

-

-

-
-

-

-
-

-
-
-
-

-
1

-

-
-

-
-

-

-
-

-    {% endraw %}
-
-    {% raw %}
-    
-

-

-
-

-    

-
model()
+
model = mc()
+model
 

 
     

@@ -341,7 +177,7 @@ 
How to use 
 
Sequential(
-  model Net
+  MC
   (stem): Sequential(
     (conv_0): ConvLayer(
       (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
@@ -529,8 +365,8 @@ 
How to use 
     

-
model.expansion = 4
-model.layers = [3,4,6,3]
+
mc.expansion = 4
+mc.layers = [3,4,6,3]
 

 
     

@@ -554,7 +390,7 @@ 
How to use 
     

-
model.body
+
mc.body
 

 
     

@@ -577,60 +413,27 @@ 
How to use How to use How to use How to use More modification.
     

-
model = Net(name='MxResNet')
+
mc = ModelConstructor(name='MxResNet')
 

 
     

@@ -969,7 +609,7 @@ 
More modification.
     

-
model.stem_sizes = [3,32,64,64]
+
mc.stem_sizes = [3,32,64,64]
 

 
     

@@ -983,7 +623,7 @@ 
More modification.
 
Now lets change activation function to Mish.
 Here is link to forum disscussion https://forums.fast.ai/t/meet-mish-new-activation-function-possible-successor-to-relu

-Mish is in model_constructor.activations

+We'v got Mish is in model_constructor.activations, but from pytorch 1.9 take it from torch:

 
 

 

@@ -995,7 +635,7 @@ 
More modification.
     

-
from model_constructor.activations import Mish
+
from torch.nn import Mish
 

 
     

@@ -1012,7 +652,7 @@ 
More modification.
     

-
model.act_fn = Mish()
+
mc.act_fn = Mish()
 

 
     

@@ -1029,7 +669,7 @@ 
More modification.
     

-
model
+
mc
 

 
     

@@ -1046,9 +686,9 @@ 
More modification.
 
MxResNet constructor
   c_in: 3, c_out: 1000
-  expansion: 1, groups: 1, dw: False
+  expansion: 1, groups: 1, dw: False, div_groups: None
   sa: False, se: False
-  stem sizes: [3, 32, 64, 64], stide on 0
+  stem sizes: [3, 32, 32, 64], stride on 0
   body sizes [64, 128, 256, 512]
   layers: [2, 2, 2, 2]

 

@@ -1068,7 +708,7 @@ 
More modification.
     

-
model()
+
mc()
 

 
     

@@ -1087,7 +727,7 @@ 
More modification.
 
Sequential(
-  model MxResNet
+  MxResNet
   (stem): Sequential(
     (conv_0): ConvLayer(
       (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
@@ -1095,12 +735,12 @@ 
More modification.MXResNet 

 
 

     

-
model.expansion = 4
-model.layers = [3,4,6,3]
-model.name = 'mxresnet50'
+
mc.expansion = 4
+mc.layers = [3,4,6,3]
+mc.name = 'mxresnet50'
 

 
     

@@ -1308,7 +948,7 @@ 
MXResNet 

 
 

     

-
model
+
mc
 

 
     

@@ -1325,9 +965,9 @@ 
MXResNet 

 

 
mxresnet50 constructor
   c_in: 3, c_out: 1000
-  expansion: 4, groups: 1, dw: False
+  expansion: 4, groups: 1, dw: False, div_groups: None
   sa: False, se: False
-  stem sizes: [3, 32, 64, 64], stide on 0
+  stem sizes: [3, 32, 32, 64], stride on 0
   body sizes [64, 128, 256, 512]
   layers: [3, 4, 6, 3]

 

@@ -1347,7 +987,7 @@ 
MXResNet 

 
 

     

-
model.stem.conv_1
+
mc.stem.conv_1
 

 
     

@@ -1366,8 +1006,8 @@ 
MXResNet 

 
 

 
ConvLayer(
-  (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
-  (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+  (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+  (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
   (act_fn): Mish()
 )

 

@@ -1388,7 +1028,7 @@ 
MXResNet 

 
 

     

-
model.body.l_0.bl_0
+
mc.body.l_0.bl_0
 

 
     

@@ -1477,7 +1117,7 @@ 
YaResNet 

 
 

     

-
model.block = YaResBlock
+
mc.block = YaResBlock
 

 
     

@@ -1501,8 +1141,8 @@ 
YaResNet 

 
 

     

-
model.name = 'YaResNet'
-model
+
mc.name = 'YaResNet'
+mc
 

 
     

@@ -1522,9 +1162,9 @@ 
YaResNet 

 

 
YaResNet constructor
   c_in: 3, c_out: 1000
-  expansion: 4, groups: 1, dw: False
+  expansion: 4, groups: 1, dw: False, div_groups: None
   sa: False, se: False
-  stem sizes: [3, 32, 64, 64], stide on 0
+  stem sizes: [3, 32, 32, 64], stride on 0
   body sizes [64, 128, 256, 512]
   layers: [3, 4, 6, 3]

 

@@ -1552,7 +1192,7 @@ 
YaResNet 

 
 

     

-
model.body.l_1.bl_0
+
mc.body.l_1.bl_0
 

 
     

@@ -1605,6 +1245,92 @@ 
YaResNet 

 

     {% endraw %}
 
+

+

+
First version 

+

+

+

+

+

+
First version, it deprecated, but still here for compatability.

+
+

+

+

+    {% raw %}
+    
+

+

+
+

+    

+
from model_constructor.net import Net
+

+
+    

+

+

+
+

+    {% endraw %}
+
+    {% raw %}
+    
+

+

+
+

+    

+
mc = Net()
+

+
+    

+

+

+
+

+    {% endraw %}
+
+    {% raw %}
+    
+

+

+
+

+    

+
mc
+

+
+    

+

+

+
+

+

+
+

+
+
+
+

+
Net constructor
+  c_in: 3, c_out: 1000
+  expansion: 1, groups: 1, dw: False, div_groups: None
+  sa: False, se: False
+  stem sizes: [3, 32, 32, 64], stride on 0
+  body sizes [64, 128, 256, 512]
+  layers: [2, 2, 2, 2]

+

+
+

+
+

+

+
+

+    {% endraw %}
+
 

  
 
diff --git a/docs/sidebar.json b/docs/sidebar.json
index 0f32017..6aa0196 100644
--- a/docs/sidebar.json
+++ b/docs/sidebar.json
@@ -1,13 +1,14 @@
 {
   "model_constructor": {
     "Overview": "/",
-    "Model constructor class.": "Net.html",
+    "Model constructor.": "ModelConstructor.html",
     "Activations functions.": "activations.html",
     "Layers": "layers.html",
     "MXResNet.": "MXResNet.html",
     "YaResNet.": "YaResNet.html",
     "Twist.": "Twist.html",
     "ConvMixer": "ConvMixer.html",
+    "First version. Net.": "Net.html",
     "Base constructor": "base_constructor.html",
     "xresnet": "xresnet.html"
   }
diff --git a/index.ipynb b/index.ipynb
deleted file mode 100644
index 31356b0..0000000
--- a/index.ipynb
+++ /dev/null
@@ -1,94 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#hide\n",
-    "from your_lib.core import *"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Project name here\n",
-    "\n",
-    "> Summary description here."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "This file will become your README and also the index of your documentation."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Install"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "`pip install your_project_name`"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## How to use"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Fill me in please! Don't forget code examples:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "2"
-      ]
-     },
-     "execution_count": null,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "1+1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}