class NewResBloc
Submodules assigned in this way will be registered, and will have their
parameters converted too when you call :meth:to, etc.
:ivar training: Boolean represents whether this module is in training or + evaluation mode. +:vartype training: bool
@@ -1090,6 +1093,9 @@class ResBlockTwist
Submodules assigned in this way will be registered, and will have their
parameters converted too when you call :meth:to, etc.
:ivar training: Boolean represents whether this module is in training or + evaluation mode. +:vartype training: bool
diff --git a/docs/_config.yml b/docs/_config.yml deleted file mode 100644 index 74610e8..0000000 --- a/docs/_config.yml +++ /dev/null @@ -1,64 +0,0 @@ -repository: ayasyrev/model_constructor -output: web -topnav_title: model_constructor -site_title: model_constructor -company_name: Andrei Yasyrev -description: Constructor for pytorch models. -# Set to false to disable KaTeX math -use_math: true -# Add Google analytics id if you have one and want to use it here -google_analytics: -# See http://nbdev.fast.ai/search for help with adding Search -google_search: - -host: 127.0.0.1 -# the preview server used. Leave as is. -port: 4000 -# the port where the preview is rendered. - -exclude: - - .idea/ - - .gitignore - - vendor - -exclude: [vendor] - -highlighter: rouge -markdown: kramdown -kramdown: - input: GFM - auto_ids: true - hard_wrap: false - syntax_highlighter: rouge - -collections: - tooltips: - output: false - -defaults: - - - scope: - path: "" - type: "pages" - values: - layout: "page" - comments: true - search: true - sidebar: home_sidebar - topnav: topnav - - - scope: - path: "" - type: "tooltips" - values: - layout: "page" - comments: true - search: true - tooltip: true - -sidebars: -- home_sidebar -permalink: pretty - -theme: jekyll-theme-cayman -baseurl: /model_constructor/ \ No newline at end of file diff --git a/docs/_data/alerts.yml b/docs/_data/alerts.yml deleted file mode 100644 index 157e162..0000000 --- a/docs/_data/alerts.yml +++ /dev/null @@ -1,15 +0,0 @@ -tip: 'MXResNet model, disscussed at https://forums.fast.ai/t/how-we-beat-the-5-epoch-imagewoof-leaderboard-score-some-new-techniques-to-consider
+ +mxresnet = Net(stem_sizes = [3, 32, 64, 64], name='MXResNet', act_fn=Mish())
+mxresnet
+mxresnet.block_sizes, mxresnet.layers
+mxresnet.stem
+mxresnet.body
+mxresnet.body
+mxresnet.head
+model = mxresnet50(c_out=10)
+model
+model.c_out, model.layers
+from nbdev import docs, showdoc, show_doc
+ResBlock(1,64,64,sa=True)
+ResBlock(4,64,64,sa=True, dw=True)
+ResBlock(4,64,64,sa=True, groups=4)
+ResBlock(2,64,64,act_fn=nn.LeakyReLU(), bn_1st=False)
+ResBlock(2, 64, 64, sa=True, se=True)
+NewResBlock now is YaResBlock - Yet Another ResNet Block! It is now at model_constructor.yaresnet. +Here i left old name for compatibility with existing Notebooks.
+ +NewResBlock(4, 64, 128, dw=1)
+model = Net()
+model
+model._block_sizes
+model.block_sizes
+model._block_sizes = [128, 256, 512, 1024]
+model
+model.block_sizes
+model.block_sizes
+model = Net()
+model.stem
+model.stem_stride_on = 1
+model.stem
+model.bn_1st = False
+model.act_fn =nn.LeakyReLU(inplace=True)
+model.sa = True
+model.se = True
+model.body.l_1
+model.block = NewResBlock
+model.expansion = 4
+m = model()
+m.body
+# %nbdev_export
+# # me = sys.modules[__name__]
+# # for n,e,l in [[ 18 , 1, [2,2,2 ,2] ],
+# # [ 34 , 1, [3,4,6 ,3] ],
+# # [ 50 , 4, [3,4,6 ,3] ],
+# # [ 101, 4, [3,4,23,3] ],
+# # [ 152, 4, [3,8,36,3] ],]:
+# # name = f'net{n}'
+# # setattr(me, name, partial(Net, expansion=e, layers=l, name=name))
+# net34 = partial(Net, expansion=1, layers=[3, 4, 6, 3], name='xresnet34')
+# net50 = partial(Net, expansion=4, layers=[3, 4, 6, 3], name='xresnet50')
+
+
+class XResNet34(Net):
+ def __init__(self):
+ super().__init__()
+ self.layers = [3,4,6,3]
+model = XResNet34()
+model
+from dataclasses import field
+@dataclass
+class ConfXresnet34:
+ name: str = 'xresnet34'
+ layers: list = field(default_factory=lambda : [3,4,6,3])
+asdict(ConfXresnet34())
+@dataclass
+class Xres50(ConfXresnet34):
+ name = 'xresnet50'
+ expansion: int = 4
+Xres50()
+asdict(Xres50())
+model = Net(asdict(Xres50()))
+model
+ConvTwist(64,64)
+ConvTwist.twist, ConvTwist.permute
+ConvTwist.use_groups, ConvTwist.groups_ch
+ConvTwist(64,64)
+ConvTwist.twist = True
+ConvTwist.permute = False
+ConvTwist(64,64)
+ConvLayerTwist(64,64, stride=1)
+ConvLayer.Conv2d
+ConvLayerTwist.Conv2d
+conv_layer = ConvLayerTwist(32, 64)
+conv_layer
+ConvTwist.twist = False
+conv_layer = ConvLayerTwist(32, 64)
+conv_layer
+conv_layer = ConvLayerTwist(32, 64, act=False)
+conv_layer
+conv_layer = ConvLayerTwist(32, 64, bn_layer=False)
+conv_layer
+conv_layer = ConvLayerTwist(32, 64, bn_1st=True)
+conv_layer
+conv_layer = ConvLayerTwist(32, 64, bn_1st=True, act_fn=nn.LeakyReLU())
+conv_layer
+conv_layer = ConvLayerTwist(32, 64, ks=1)
+conv_layer
+conv_layer = ConvLayerTwist(32, 64, ks=1, stride=2)
+conv_layer
+conv_layer = ConvLayerTwist(32, 64, stride=2)
+conv_layer
+ConvTwist.groups_ch = 4
+conv_layer = ConvLayerTwist(32, 64, stride=2)
+conv_layer
+bl = NewResBlockTwist(4,64,64,sa=True)
+bl
+bl = NewResBlockTwist(4,64,64,stride=2)
+bl
+bl = NewResBlockTwist(4,64,128,stride=2)
+bl
+bl = ResBlockTwist(4,64,64,sa=True)
+bl
+bl = ResBlockTwist(4,64,64,stride=2)
+bl
+bl = ResBlockTwist(4,64,128,stride=2)
+bl
+model = Net(expansion=4, layers=[3,4,6,3])
+model.block = NewResBlockTwist
+model.body
+model.block = ResBlockTwist
+m = model()
+m.stem
+m.head
+m.body.l_0
+m.body.l_1
+m.body.l_2
+m.body.l_3
+yaresnet = Net(block=YaResBlock, stem_sizes = [3, 32, 64, 64], name='YaResNet')
+yaresnet
+yaresnet.block_sizes, yaresnet.layers
+yaresnet.stem
+yaresnet.body
+yaresnet.head
+# bs_test = 16
+# xb = torch.randn(bs_test, 3, 128, 128)
+# y = yaresnet()(xb)
+# print(y.shape)
+# assert y.shape == torch.Size([bs_test, 1000]), f"size"
+Lots of experiments showed that it worth trying Mish activation function.
+ +yaresnet.act_fn = Mish()
+yaresnet()
+model = yaresnet50(c_out=10)
+model
+model.c_out, model.layers
+Activation functions, forked from https://github.com/rwightman/pytorch-image-models/timm/models/layers/activations.py
+ +Mish: Self Regularized
+Non-Monotonic Activation Function
+https://github.com/digantamisra98/Mish
+fastai forum discussion https://forums.fast.ai/t/meet-mish-new-activation-function-possible-successor-to-relu
class Stem
class Stem
class Stem
class Stem
class Stem
class Stem
class Stem
- {% raw %}
-DownsampleBlock
-class BasicBlock[source]+
BasicBlock(ni,nf,expansion=1,stride=1,zero_bn=False,conv_layer=ConvLayer,act_fn=ReLU(inplace=True),downsample_block=DownsampleBlock, **kwargs) ::Module
classBasicBlock[source]
BasicBlock(ni,nf,expansion=1,stride=1,zero_bn=False,conv_layer=ConvLayer,act_fn=ReLU(inplace=True),downsample_block=DownsampleBlock, **kwargs) ::ModuleBasic block (simplified) as in pytorch resnet
@@ -462,16 +436,16 @@
classBasicBlock
class BasicBlock
-class Bottleneck[source]+
Bottleneck(ni,nh,expansion=4,stride=1,zero_bn=False,conv_layer=ConvLayer,act_fn=ReLU(inplace=True),downsample_block=DownsampleBlock, **kwargs) ::Module
classBottleneck[source]
Bottleneck(ni,nh,expansion=4,stride=1,zero_bn=False,conv_layer=ConvLayer,act_fn=ReLU(inplace=True),downsample_block=DownsampleBlock, **kwargs) ::ModuleBottlneck block for resnet models
@@ -494,16 +468,16 @@
classBottleneck
class Bottleneck
-class BasicLayer[source]+
BasicLayer(block,blocks,ni,nf,expansion,stride,sa=False, **kwargs) ::Sequential
classBasicLayer[source]
BasicLayer(block,blocks,ni,nf,expansion,stride,sa=False, **kwargs) ::SequentialLayer from blocks
@@ -526,7 +500,14 @@
classBasicLayer
class Body[source]+
Body(block,body_in=64,body_out=512,bodylayer=BasicLayer,expansion=1,layer_szs=[64, 128, 256],blocks=[2, 2, 2, 2],sa=False, **kwargs) ::Sequential
classBody[source]
Body(block,body_in=64,body_out=512,bodylayer=BasicLayer,expansion=1,layer_szs=[64, 128, 256],blocks=[2, 2, 2, 2],sa=False, **kwargs) ::SequentialConstructor for body
@@ -564,9 +538,16 @@
classBody+ +
class Body
- {% raw %}
-class Head
- {% raw %}
-init_model
-class Net[source]+
Net(stem=Stem,body=Body,block=BasicBlock,sa=False,layer_szs=[64, 128, 256],blocks=[2, 2, 2, 2],head=Head,c_in=3,num_classes=1000,body_in=64,body_out=512,expansion=1,init_fn=init_model, **kwargs) ::Sequential
classNet[source]
Net(stem=Stem,body=Body,block=BasicBlock,sa=False,layer_szs=[64, 128, 256],blocks=[2, 2, 2, 2],head=Head,c_in=3,num_classes=1000,body_in=64,body_out=512,expansion=1,init_fn=init_model, **kwargs) ::SequentialConstructor for model
@@ -939,9 +913,16 @@
classNet+ +
class Net
class Net
class Netpip install model-constructor
Or install from repo:
+ +pip install git+https://github.com/ayasyrev/model_constructor.git
First import constructor class, then create model constructor oject.
+Now you can change every part of model.
+ +from model_constructor.net import *
+model = Net()
+model
+Now we have model consructor, default setting as xresnet18. And we can get model after call it.
+ +model.c_in
+model.c_out
+model.stem_sizes
+model.layers
+model.expansion
+model()
+If you want to change model, just change constructor parameters.
+Lets create xresnet50.
model.expansion = 4
+model.layers = [3,4,6,3]
+Now we can look at model body and if we call constructor - we have pytorch model!
+ +model.body
+Main purpose of this module - fast and easy modify model. +And here is the link to more modification to beat Imagenette leaderboard with add MaxBlurPool and modification to ResBlock https://github.com/ayasyrev/imagenette_experiments/blob/master/ResnetTrick_create_model_fit.ipynb
+But now lets create model as mxresnet50 from fastai forums tread https://forums.fast.ai/t/how-we-beat-the-5-epoch-imagewoof-leaderboard-score-some-new-techniques-to-consider
+ +Lets create mxresnet constructor.
+ +model = Net(name='MxResNet')
+Then lets modify stem.
+ +model.stem_sizes = [3,32,64,64]
+Now lets change activation function to Mish.
+Here is link to forum disscussion https://forums.fast.ai/t/meet-mish-new-activation-function-possible-successor-to-relu
+Mish is in model_constructor.activations
from model_constructor.activations import Mish
+model.act_fn = Mish()
+model
+model()
+Now lets make MxResNet50
+ +model.expansion = 4
+model.layers = [3,4,6,3]
+model.name = 'mxresnet50'
+Now we have mxresnet50 constructor.
+We can inspect every parts of it.
+And after call it we got model.
model
+model.stem.conv_1
+model.body.l_0.bl_0
+Now lets change Resblock to YaResBlock (Yet another ResNet, former NewResBlock) is in lib from version 0.1.0
+ +from model_constructor.yaresnet import YaResBlock
+model.block = YaResBlock
+That all. Now we have YaResNet constructor
+ +model.name = 'YaResNet'
+model
+Let see what we have.
+ +model.body.l_1.bl_0
+conv_layer = ConvLayer(32, 64)
+conv_layer
+conv_layer = ConvLayer(32, 64, act=False)
+conv_layer
+conv_layer = ConvLayer(32, 64, bn_layer=False)
+conv_layer
+conv_layer = ConvLayer(32, 64, bn_1st=True)
+conv_layer
+conv_layer = ConvLayer(32, 64, bn_1st=True, act_fn=nn.LeakyReLU())
+conv_layer
+se_block = SEBlock(128)
+se_block
+se_block = SEBlockConv(128)
+se_block
+body = Body(Bottleneck, expansion=4)
-body
+body = Body(XResBlock, expansion=4)
+body
resnet18[source]+
resnet18(**kwargs)
xresnet18[source]-
xresnet18(**kwargs)Constructs a ResNet-18 model.
+Constructs a xresnet-18 model.
resnet18
@@ -386,9 +439,9 @@ resnet18
-resnet34[source]+
resnet34(**kwargs)
xresnet34[source]-
xresnet34(**kwargs)Constructs a ResNet-34 model.
+Constructs axresnet-34 model.
resnet34
@@ -411,9 +464,9 @@ resnet34
-resnet50[source]+
resnet50(**kwargs)
xresnet50[source]-
xresnet50(**kwargs)Constructs a ResNet-18 model.
+Constructs axresnet-34 model.
resnet50
+
+resnet18()
+xresnet18()
resnet50
Net(
(stem): Stem(
- sizes: [3, 64]
- (conv0): ConvLayer(
- (conv): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+ sizes: [3, 32, 32, 64]
+ (conv_0): ConvLayer(
+ (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+ (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+ (act_fn): ReLU(inplace=True)
+ )
+ (conv_1): ConvLayer(
+ (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+ (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+ (act_fn): ReLU(inplace=True)
+ )
+ (conv_2): ConvLayer(
+ (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(act_fn): ReLU(inplace=True)
)
@@ -460,8 +530,8 @@ resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50
-model = resnet34()
-model
+xresnet34()
@@ -652,9 +727,19 @@ resnet50
Net(
(stem): Stem(
- sizes: [3, 64]
- (conv0): ConvLayer(
- (conv): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+ sizes: [3, 32, 32, 64]
+ (conv_0): ConvLayer(
+ (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+ (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+ (act_fn): ReLU(inplace=True)
+ )
+ (conv_1): ConvLayer(
+ (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+ (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+ (act_fn): ReLU(inplace=True)
+ )
+ (conv_2): ConvLayer(
+ (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(act_fn): ReLU(inplace=True)
)
@@ -663,8 +748,8 @@ resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50resnet50
-xb = torch.randn(16, 3, 128, 128)
-y = model(xb)
-y.shape
+model = xresnet50()
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -991,26 +1115,9 @@ resnet50
-
-
-
-
-model = resnet50()
-
-
-
-
-
-
-
- {% endraw %}
- {% raw %}
+
@@ -1035,366 +1142,373 @@ resnet50resnet50
-xb = torch.randn(16, 3, 128, 128)
+model.head
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+xb = torch.randn(8, 3, 128, 128)
y = model(xb)
y.shape
@@ -1432,7 +1583,7 @@ resnet50
-torch.Size([16, 1000])
+torch.Size([8, 1000])
@@ -1441,11 +1592,11 @@ resnet50
diff --git a/docs/activations.html b/docs/activations.html
index 74137ea..488bdf8 100644
--- a/docs/activations.html
+++ b/docs/activations.html
@@ -141,7 +141,7 @@ MishJit
@@ -225,7 +228,7 @@ MishJitMe - memory-efficient.
-ScriptFunction object at 0x7f60a962c950>[source]
ScriptFunction object at 0x7f60a962c950>()
+ScriptFunction object at 0x7f691998d310>[source]
ScriptFunction object at 0x7f691998d310>()
@@ -249,7 +252,7 @@ Scrip
@@ -273,7 +276,7 @@ Scrip