基于拟合的优化尝试.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from benchmark_funs import *\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.utils.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm\n",
    "from pytorch_lamb import Lamb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(model, data, fun):\n",
    "    criterion = nn.MSELoss()\n",
    "    # optimizer = optim.LBFGS(model.parameters(), lr=1e-1)\n",
    "    # optimizer = optim.AdamW(model.parameters(), lr=3e-1)\n",
    "    # optimizer = optim.AdamW(model.parameters(), lr=3e-1)\n",
    "    optimizer = Lamb(model.parameters(), lr=3e-0)\n",
    "    dataset = torch.utils.data.TensorDataset(data, fun(data))\n",
    "    loader = torch.utils.data.DataLoader(dataset, batch_size=10000, shuffle=True)\n",
    "    for epoch in range(3):\n",
    "        box = tqdm.tqdm(enumerate(loader), desc=f\"正在拟合{fun.name}\")\n",
    "        for batch_idx, (data, target) in box:\n",
    "            def closure():\n",
    "                pred_output = model(data)\n",
    "                loss = criterion(pred_output.squeeze(), target.squeeze())\n",
    "                box.set_postfix(loss=loss.detach().numpy().item())\n",
    "                loss.backward()\n",
    "                return loss\n",
    "            closure()\n",
    "            optimizer.step()\n",
    "            # optimizer.step(closure=closure)\n",
    "            optimizer.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "def evaluate(opti, p_ind=1000):\n",
    "    s = 0\n",
    "    for fun in benchmark_functions:\n",
    "        assert fun.larger_better == False # 最小化\n",
    "        interval_len = fun.ub - fun.lb\n",
    "        # 第一步，拟合 model\n",
    "        # 万能近似定理：单隐层使用任意压缩函数的前馈神经网络只要隐层数量足够多就能够以任意精度逼近任意可测函数。\n",
    "        # https://blog.csdn.net/qq_37983752/article/details/115055707\n",
    "        # model = nn.Sequential(\n",
    "        #     nn.Linear(fun.dimension, 10),\n",
    "        #     nn.Sigmoid(), # 压缩函数，将值压缩到0-1之间的连续可导函数\n",
    "        #     # nn.Linear(1000, 100),  \n",
    "        #     # nn.Sigmoid(),\n",
    "        #     nn.Linear(10, 1),    \n",
    "        # )\n",
    "        \n",
    "        model = nn.Sequential(\n",
    "            nn.Linear(fun.dimension, 10000),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm1d(10000), \n",
    "            nn.Linear(10000, 1000),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm1d(1000), \n",
    "            nn.Linear(1000, 100),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm1d(100), \n",
    "            nn.Linear(100, 10),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm1d(10),\n",
    "            nn.Linear(10, 1),    \n",
    "        )\n",
    "        X = nn.init.trunc_normal_(torch.empty(fun.get_budget(), fun.dimension), fun.lb+interval_len/2, interval_len/6, a=fun.lb, b=fun.ub)\n",
    "        fit(model, X, fun)\n",
    "        # 第二步，优化 model\n",
    "        X = nn.init.trunc_normal_(torch.empty(p_ind, fun.dimension), fun.lb+interval_len/2, interval_len/6, a=fun.lb, b=fun.ub)\n",
    "        X = X.to(device)\n",
    "        X.requires_grad_(True)\n",
    "        optimizer = opti([X])\n",
    "        # box = tqdm.tqdm(range(1000), desc=f\"正在优化{fun.name}\")\n",
    "        box = tqdm.tqdm(range(fun.get_budget()//p_ind), desc=f\"正在优化{fun.name}的代理模型。\")\n",
    "        best_loss = 1e10\n",
    "        patience = 0\n",
    "        for i in box:\n",
    "            # X[X<fun.lb] = fun.lb\n",
    "            # X[X>fun.ub] = fun.ub\n",
    "            loss = fun(X).min() - fun.optival\n",
    "            # loss = fun(X).sum()\n",
    "            loss_item = loss.detach().numpy().item()\n",
    "            box.set_postfix(loss=loss_item)\n",
    "            if loss_item<best_loss:\n",
    "                patience = 0\n",
    "                best_loss = loss_item\n",
    "            else:\n",
    "                patience +=1\n",
    "                if patience>100:\n",
    "                    break\n",
    "            loss = model(X).min() - fun.optival\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            if X.grad.norm()<1e-8:\n",
    "                break\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "        s+=loss.detach().numpy().item()\n",
    "    return s\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "正在拟合Sphere: 30it [00:13,  2.23it/s, loss=1.02e+9]\n",
      "正在拟合Sphere: 30it [00:13,  2.26it/s, loss=9.66e+8]\n",
      "正在拟合Sphere: 30it [00:13,  2.26it/s, loss=9.02e+8]\n",
      "正在优化Sphere的代理模型。:   3%|▎         | 807/30000 [00:10<06:12, 78.35it/s, loss=1.62e+4]\n",
      "正在拟合Rosenbrock: 13it [00:05,  2.27it/s, loss=7.55e+15]"
     ]
    }
   ],
   "source": [
    "opti = lambda p: optim.Adam(p, lr=0.1)\n",
    "evaluate(opti, p_ind=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch2.0",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}