From 0c963053f4d1658cd510227fcc13bd59f1e37261 Mon Sep 17 00:00:00 2001 From: Ultralytics AI Assistant <135830346+UltralyticsAssistant@users.noreply.github.com> Date: Mon, 1 Jan 2024 02:56:49 +0100 Subject: [PATCH] Add Ultralytics actions workflow in `format.yml` (#10) Co-authored-by: glenn-jocher --- .github/workflows/format.yml | 20 +++++++ gcp/wave_pytorch_gcp.py | 103 +++++++++++++++++++++-------------- train.py | 89 ++++++++++++++++-------------- train_tf.py | 65 ++++++++++++---------- utils/torch_utils.py | 18 +++--- utils/utils.py | 38 +++++++------ 6 files changed, 198 insertions(+), 135 deletions(-) create mode 100644 .github/workflows/format.yml diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 0000000..053645d --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,20 @@ +# Ultralytics 🚀, AGPL-3.0 license +# Ultralytics Format Workflow +# This workflow automatically formats code and documentation in pull requests and pushes to main branch + +name: Ultralytics Actions + +on: + push: + branches: [main,master] + pull_request: + branches: [main,master] + +jobs: + format: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: Run Ultralytics Formatting Actions + uses: ultralytics/actions@main diff --git a/gcp/wave_pytorch_gcp.py b/gcp/wave_pytorch_gcp.py index 86d077b..ba3c85c 100644 --- a/gcp/wave_pytorch_gcp.py +++ b/gcp/wave_pytorch_gcp.py @@ -9,33 +9,33 @@ # set printoptions torch.set_printoptions(linewidth=320, precision=8) -np.set_printoptions(linewidth=320, formatter={'float_kind': '{:11.5g}'.format}) # format short g, %precision=5 +np.set_printoptions(linewidth=320, formatter={"float_kind": "{:11.5g}".format}) # format short g, %precision=5 -pathd = 'data/' -pathr = 'results/' +pathd = "data/" +pathr = "results/" torch.manual_seed(1) + def runexample(H, model, str, lr=0.001, amsgrad=False): epochs = 100000 validations = 5000 printInterval = 1000 # batch_size = 10000 - data = 'wavedata25ns.mat' + data = "wavedata25ns.mat" cuda = torch.cuda.is_available() - os.makedirs(pathr + 'models', exist_ok=True) - name = (data[:-4] + '%s%glr%s' % (H[:], lr, str)).replace(', ', '.').replace('[', '_').replace(']', '_') + os.makedirs(pathr + "models", exist_ok=True) + name = (data[:-4] + "%s%glr%s" % (H[:], lr, str)).replace(", ", ".").replace("[", "_").replace("]", "_") tica = time.time() - device = torch.device('cuda:0' if cuda else 'cpu') - print('Running %s on %s\n%s' % - (name, device.type, torch.cuda.get_device_properties(0) if cuda else '')) + device = torch.device("cuda:0" if cuda else "cpu") + print("Running %s on %s\n%s" % (name, device.type, torch.cuda.get_device_properties(0) if cuda else "")) if not os.path.isfile(pathd + data): - os.system('wget -P data/ https://storage.googleapis.com/ultralytics/' + data) + os.system("wget -P data/ https://storage.googleapis.com/ultralytics/" + data) mat = scipy.io.loadmat(pathd + data) - x = mat['inputs'] # inputs (nx512) [waveform1 waveform2] - y = mat['outputs'][:, 1:2] # outputs (nx4) [position(mm), time(ns), PE, E(MeV)] + x = mat["inputs"] # inputs (nx512) [waveform1 waveform2] + y = mat["outputs"][:, 1:2] # outputs (nx4) [position(mm), time(ns), PE, E(MeV)] nz, nx = x.shape ny = y.shape[1] @@ -43,7 +43,7 @@ def runexample(H, model, str, lr=0.001, amsgrad=False): y, ymu, ys = normalize(y, 0) # normalize each output column x, y = torch.Tensor(x), torch.Tensor(y) x, y, xv, yv, xt, yt = splitdata(x, y, train=0.70, validate=0.15, test=0.15, shuffle=True) - labels = ['train', 'validate', 'test'] + labels = ["train", "validate", "test"] print(model) if cuda: @@ -57,7 +57,7 @@ def runexample(H, model, str, lr=0.001, amsgrad=False): ticb = time.time() L = np.full((epochs, 3), np.nan) - best = (0, 1E6, model.state_dict()) # best (epoch, validation loss, model) + best = (0, 1e6, model.state_dict()) # best (epoch, validation loss, model) for i in range(epochs): y_pred = model(x) y_predv = model(xv) @@ -72,13 +72,13 @@ def runexample(H, model, str, lr=0.001, amsgrad=False): if L[i, 1] < best[1]: best = (i, L[i, 1], copy.deepcopy(model.state_dict())) if (i - best[0]) > validations: - print('\n%g validation checks exceeded at epoch %g.' % (validations, i)) + print("\n%g validation checks exceeded at epoch %g." % (validations, i)) break if i % printInterval == 0: # print and save progress # scipy.io.savemat(pathr + name + '.mat', dict(bestepoch=best[0], loss=L[best[0]], L=L, name=name)) _, std = stdpt(y_predv - yv, ys) - print('%.3fs' % (time.time() - ticb), i, L[i], std) + print("%.3fs" % (time.time() - ticb), i, L[i], std) ticb = time.time() # Zero gradients, perform a backward pass, and update the weights. @@ -86,16 +86,16 @@ def runexample(H, model, str, lr=0.001, amsgrad=False): loss.backward() optimizer.step() else: - print('WARNING: Validation loss still decreasing after %g epochs (train longer).' % (i + 1)) - #torch.save(best[2], pathr + 'models/' + name + '.pt') + print("WARNING: Validation loss still decreasing after %g epochs (train longer)." % (i + 1)) + # torch.save(best[2], pathr + 'models/' + name + '.pt') model.load_state_dict(best[2]) dt = time.time() - tica - print('\nFinished %g epochs in %.3fs (%.3f epochs/s)\nBest results from epoch %g:' % (i + 1, dt, i / dt, best[0])) + print("\nFinished %g epochs in %.3fs (%.3f epochs/s)\nBest results from epoch %g:" % (i + 1, dt, i / dt, best[0])) loss, std = np.zeros(3), np.zeros((3, ny)) for i, (xi, yi) in enumerate(((x, y), (xv, yv), (xt, yt))): loss[i], std[i] = stdpt(model(xi) - yi, ys) - print('%.5f %s %s' % (loss[i], std[i, :], labels[i])) + print("%.5f %s %s" % (loss[i], std[i, :], labels[i])) # scipy.io.savemat(pathr + name + '.mat', dict(bestepoch=best[0], loss=loss, std=std, L=L, name=name)) # files.download(pathr + name + '.mat') @@ -111,6 +111,8 @@ def runexample(H, model, str, lr=0.001, amsgrad=False): H = [512, 64, 8, 1] + + class LinearAct(torch.nn.Module): def __init__(self, nx, ny): super(LinearAct, self).__init__() @@ -118,7 +120,8 @@ def __init__(self, nx, ny): self.act = torch.nn.Tanh() def forward(self, x): - return self.act(self.Linear1(x)) + return self.act(self.Linear1(x)) + class WAVE(torch.nn.Module): def __init__(self, n): # n = [512, 108, 23, 5, 1] @@ -133,16 +136,17 @@ def forward(self, x): def tsact(): # TS activation function H = [512, 64, 8, 1] - tsv = ['Sigmoid']#['Tanh', 'LogSigmoid', 'Softsign', 'ELU'] + tsv = ["Sigmoid"] # ['Tanh', 'LogSigmoid', 'Softsign', 'ELU'] # tsv = np.logspace(-4,-2,11) tsy = [] for a in tsv: + class LinearAct(torch.nn.Module): def __init__(self, nx, ny): super(LinearAct, self).__init__() self.Linear1 = torch.nn.Linear(nx, ny) - self.act = eval('torch.nn.' + a + '()') + self.act = eval("torch.nn." + a + "()") def forward(self, x): return self.act(self.Linear1(x)) @@ -158,16 +162,18 @@ def forward(self, x): return self.fc2(self.fc1(self.fc0(x))) for i in range(10): - tsy.append(runexample(H, model=WAVE(H), str=('.' + a))) - scipy.io.savemat(pathr + 'TS.sigmoid' + '.mat', dict(tsv=tsv, tsy=np.array(tsy))) + tsy.append(runexample(H, model=WAVE(H), str=("." + a))) + scipy.io.savemat(pathr + "TS.sigmoid" + ".mat", dict(tsv=tsv, tsy=np.array(tsy))) + def tsnoact(): # TS activation function H = [512, 64, 8, 1] - tsv = ['NoAct']#['Tanh', 'LogSigmoid', 'Softsign', 'ELU'] + tsv = ["NoAct"] # ['Tanh', 'LogSigmoid', 'Softsign', 'ELU'] # tsv = np.logspace(-4,-2,11) tsy = [] for a in tsv: + class WAVE(torch.nn.Module): def __init__(self, n): # n = [512, 108, 23, 5, 1] super(WAVE, self).__init__() @@ -179,8 +185,8 @@ def forward(self, x): return self.fc2(self.fc1(self.fc0(x))) for i in range(10): - tsy.append(runexample(H, model=WAVE(H), str=('.' + a))) - scipy.io.savemat(pathr + 'TS.noact' + '.mat', dict(tsv=tsv, tsy=np.array(tsy))) + tsy.append(runexample(H, model=WAVE(H), str=("." + a))) + scipy.io.savemat(pathr + "TS.noact" + ".mat", dict(tsv=tsv, tsy=np.array(tsy))) def tslr(): # TS learning rate @@ -188,8 +194,8 @@ def tslr(): # TS learning rate tsy = [] for a in tsv: for i in range(10): - tsy.append(runexample(H, model=WAVE(H), str=('.' + 'Tanh'), lr=a)) - scipy.io.savemat(pathr + 'TS.lr' + '.mat', dict(tsv=tsv, tsy=np.array(tsy))) + tsy.append(runexample(H, model=WAVE(H), str=("." + "Tanh"), lr=a)) + scipy.io.savemat(pathr + "TS.lr" + ".mat", dict(tsv=tsv, tsy=np.array(tsy))) def tsams(): # TS AMSgrad @@ -197,8 +203,8 @@ def tsams(): # TS AMSgrad tsy = [] for a in tsv: for i in range(3): - tsy.append( runexample(H, model=WAVE(H), str=('.TanhAMS' + str(a)), amsgrad=a) ) - scipy.io.savemat(pathr + 'TS.AMSgrad' + '.mat', dict(tsv=tsv, tsy=np.array(tsy))) + tsy.append(runexample(H, model=WAVE(H), str=(".TanhAMS" + str(a)), amsgrad=a)) + scipy.io.savemat(pathr + "TS.AMSgrad" + ".mat", dict(tsv=tsv, tsy=np.array(tsy))) def tsshape(): # TS network shape @@ -220,29 +226,36 @@ def tsshape(): # TS network shape tsy = [] H = tsv[0] + class WAVE(torch.nn.Module): def __init__(self, n): # n = [512, 108, 23, 5, 1] super(WAVE, self).__init__() self.fc0 = LinearAct(n[0], n[1]) self.fc1 = torch.nn.Linear(n[1], n[2]) + def forward(self, x): return self.fc1(self.fc0(x)) + for i in range(10): - tsy.append(runexample(H, model=WAVE(H), str=('.' + 'Tanh'))) + tsy.append(runexample(H, model=WAVE(H), str=("." + "Tanh"))) H = tsv[1] + class WAVE(torch.nn.Module): def __init__(self, n): # n = [512, 108, 23, 5, 1] super(WAVE, self).__init__() self.fc0 = LinearAct(n[0], n[1]) self.fc1 = LinearAct(n[1], n[2]) self.fc2 = torch.nn.Linear(n[2], n[3]) + def forward(self, x): return self.fc2(self.fc1(self.fc0(x))) + for i in range(10): - tsy.append(runexample(H, model=WAVE(H), str=('.' + 'Tanh'))) + tsy.append(runexample(H, model=WAVE(H), str=("." + "Tanh"))) H = tsv[2] + class WAVE(torch.nn.Module): def __init__(self, n): # n = [512, 108, 23, 5, 1] super(WAVE, self).__init__() @@ -250,12 +263,15 @@ def __init__(self, n): # n = [512, 108, 23, 5, 1] self.fc1 = LinearAct(n[1], n[2]) self.fc2 = LinearAct(n[2], n[3]) self.fc3 = torch.nn.Linear(n[3], n[4]) + def forward(self, x): return self.fc3(self.fc2(self.fc1(self.fc0(x)))) + for i in range(10): - tsy.append(runexample(H, model=WAVE(H), str=('.' + 'Tanh'))) + tsy.append(runexample(H, model=WAVE(H), str=("." + "Tanh"))) H = tsv[3] + class WAVE(torch.nn.Module): def __init__(self, n): # n = [512, 108, 23, 5, 1] super(WAVE, self).__init__() @@ -264,12 +280,15 @@ def __init__(self, n): # n = [512, 108, 23, 5, 1] self.fc2 = LinearAct(n[2], n[3]) self.fc3 = LinearAct(n[3], n[4]) self.fc4 = torch.nn.Linear(n[4], n[5]) + def forward(self, x): return self.fc4(self.fc3(self.fc2(self.fc1(self.fc0(x))))) + for i in range(10): - tsy.append(runexample(H, model=WAVE(H), str=('.' + 'Tanh'))) + tsy.append(runexample(H, model=WAVE(H), str=("." + "Tanh"))) H = tsv[4] + class WAVE(torch.nn.Module): def __init__(self, n): # n = [512, 108, 23, 5, 1] super(WAVE, self).__init__() @@ -279,13 +298,15 @@ def __init__(self, n): # n = [512, 108, 23, 5, 1] self.fc3 = LinearAct(n[3], n[4]) self.fc4 = LinearAct(n[4], n[5]) self.fc5 = torch.nn.Linear(n[5], n[6]) + def forward(self, x): return self.fc5(self.fc4(self.fc3(self.fc2(self.fc1(self.fc0(x)))))) + for i in range(10): - tsy.append(runexample(H, model=WAVE(H), str=('.' + 'Tanh'))) - scipy.io.savemat(pathr + 'TS.shape' + '.mat', dict(tsv=tsv, tsy=np.array(tsy))) + tsy.append(runexample(H, model=WAVE(H), str=("." + "Tanh"))) + scipy.io.savemat(pathr + "TS.shape" + ".mat", dict(tsv=tsv, tsy=np.array(tsy))) -if __name__ == '__main__': - #tsnoact() - tsact() \ No newline at end of file +if __name__ == "__main__": + # tsnoact() + tsact() diff --git a/train.py b/train.py index fb0b3f8..19c4529 100644 --- a/train.py +++ b/train.py @@ -11,27 +11,27 @@ ONNX_EXPORT = False -pathd = 'data/' -pathr = 'results/' -labels = ['train', 'validate', 'test'] +pathd = "data/" +pathr = "results/" +labels = ["train", "validate", "test"] torch.manual_seed(1) def train(H, model, str, lr=0.001): - data = 'wavedata25ns.mat' + data = "wavedata25ns.mat" cuda = torch.cuda.is_available() - os.makedirs(pathr + 'models', exist_ok=True) - name = (data[:-4] + '%s%glr%s' % (H[:], lr, str)).replace(', ', '.').replace('[', '_').replace(']', '_') - print('Running ' + name) + os.makedirs(pathr + "models", exist_ok=True) + name = (data[:-4] + "%s%glr%s" % (H[:], lr, str)).replace(", ", ".").replace("[", "_").replace("]", "_") + print("Running " + name) device = select_device() if not os.path.isfile(pathd + data): - os.system('wget -P data/ https://storage.googleapis.com/ultralytics/' + data) + os.system("wget -P data/ https://storage.googleapis.com/ultralytics/" + data) mat = scipy.io.loadmat(pathd + data) - x = mat['inputs'][:] # inputs (nx512) [waveform1 waveform2] - y = mat['outputs'][:, 0:2] # outputs (nx4) [position(mm), time(ns), PE, E(MeV)] + x = mat["inputs"][:] # inputs (nx512) [waveform1 waveform2] + y = mat["outputs"][:, 0:2] # outputs (nx4) [position(mm), time(ns), PE, E(MeV)] nz, nx = x.shape ny = y.shape[1] @@ -61,10 +61,11 @@ def train(H, model, str, lr=0.001): # Scheduler stopper = patienceStopper(epochs=opt.epochs, patience=24, printerval=opt.printerval) - scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=20, factor=0.1, min_lr=1E-5, - verbose=True) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( + optimizer, patience=20, factor=0.1, min_lr=1e-5, verbose=True + ) - lossv = 1E6 + lossv = 1e6 bs = opt.batch_size nb = int(np.ceil(x.shape[0] / bs)) L = np.full((opt.epochs, 3), np.nan) @@ -77,7 +78,7 @@ def train(H, model, str, lr=0.001): for bi in range(nb): j = range(bi * bs, min((bi + 1) * bs, x.shape[0])) if ONNX_EXPORT: - _ = torch.onnx._export(model, x, 'model.onnx', verbose=True) + _ = torch.onnx._export(model, x, "model.onnx", verbose=True) return loss = MSE(model(x[j]), y[j]) @@ -108,11 +109,11 @@ def train(H, model, str, lr=0.001): for i, (xi, yi) in enumerate(((x, y), (xv, yv), (xt, yt))): with torch.no_grad(): r = stopper.bestmodel(xi) - yi # residuals, ().detach? - loss[i] = (r ** 2).mean().cpu().item() + loss[i] = (r**2).mean().cpu().item() std[i] = r.std(0).cpu().numpy() * ys - print('%.5f %s %s' % (loss[i], std[i, :], labels[i])) + print("%.5f %s %s" % (loss[i], std[i, :], labels[i])) - scipy.io.savemat(pathr + name + '.mat', dict(bestepoch=stopper.bestloss, loss=loss, std=std, L=L, name=name)) + scipy.io.savemat(pathr + name + ".mat", dict(bestepoch=stopper.bestloss, loss=loss, std=std, L=L, name=name)) # files.download(pathr + name + '.mat') return np.concatenate(([stopper.bestloss], np.array(loss), np.array(std.ravel()))) @@ -140,12 +141,14 @@ def __init__(self, n_out=2): self.layer1 = nn.Sequential( nn.Conv2d(1, 32, kernel_size=(1, 9), stride=(1, 2), padding=(0, 4), bias=False), nn.BatchNorm2d(32), - nn.LeakyReLU(0.1)) + nn.LeakyReLU(0.1), + ) # nn.MaxPool2d(kernel_size=(1, 2), stride=1)) self.layer2 = nn.Sequential( nn.Conv2d(32, 64, kernel_size=(1, 9), stride=(1, 2), padding=(0, 4), bias=False), nn.BatchNorm2d(64), - nn.LeakyReLU(0.1)) + nn.LeakyReLU(0.1), + ) # nn.MaxPool2d(kernel_size=(1, 2), stride=1)) self.layer3 = nn.Conv2d(64, n_out, kernel_size=(2, 64), stride=(1, 1), padding=(0, 0)) @@ -166,17 +169,22 @@ def __init__(self, n_out=2): self.layer1 = nn.Sequential( nn.Conv2d(in_channels=2, out_channels=n, kernel_size=(1, 33), stride=(1, 2), padding=(0, 16), bias=False), nn.BatchNorm2d(n), - nn.LeakyReLU(0.1)) + nn.LeakyReLU(0.1), + ) self.layer2 = nn.Sequential( - nn.Conv2d(in_channels=n, out_channels=n * 2, kernel_size=(1, 17), stride=(1, 2), padding=(0, 8), - bias=False), + nn.Conv2d( + in_channels=n, out_channels=n * 2, kernel_size=(1, 17), stride=(1, 2), padding=(0, 8), bias=False + ), nn.BatchNorm2d(n * 2), - nn.LeakyReLU(0.1)) + nn.LeakyReLU(0.1), + ) self.layer3 = nn.Sequential( - nn.Conv2d(in_channels=n * 2, out_channels=n * 4, kernel_size=(1, 9), stride=(1, 2), padding=(0, 4), - bias=False), + nn.Conv2d( + in_channels=n * 2, out_channels=n * 4, kernel_size=(1, 9), stride=(1, 2), padding=(0, 4), bias=False + ), nn.BatchNorm2d(n * 4), - nn.LeakyReLU(0.1)) + nn.LeakyReLU(0.1), + ) self.layer4 = nn.Conv2d(n * 4, n_out, kernel_size=(1, 32), stride=1, padding=0) def forward(self, x): # x.shape = [bs, 512] @@ -200,14 +208,15 @@ def __init__(self, n_out=2): nn.Conv2d(1, 32, kernel_size=(2, 30), stride=(1, 2), padding=(1, 15), bias=False), nn.BatchNorm2d(32), nn.LeakyReLU(0.1), - nn.MaxPool2d(kernel_size=(1, 2), stride=1)) + nn.MaxPool2d(kernel_size=(1, 2), stride=1), + ) self.layer2 = nn.Sequential( nn.Conv2d(32, 64, kernel_size=(2, 30), stride=(1, 2), padding=(0, 15), bias=False), nn.BatchNorm2d(64), nn.LeakyReLU(0.1), - nn.MaxPool2d(kernel_size=(1, 2), stride=1)) - self.layer3 = nn.Sequential( - nn.Conv2d(64, n_out, kernel_size=(2, 64), stride=(1, 1), padding=(0, 0))) + nn.MaxPool2d(kernel_size=(1, 2), stride=1), + ) + self.layer3 = nn.Sequential(nn.Conv2d(64, n_out, kernel_size=(2, 64), stride=(1, 1), padding=(0, 0))) def forward(self, x): # x.shape = [bs, 512] x = x.view((-1, 2, 256)) # [bs, 2, 256] @@ -220,26 +229,26 @@ def forward(self, x): # x.shape = [bs, 512] H = [512, 64, 8, 2] -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--epochs', type=int, default=5000, help='number of epochs') - parser.add_argument('--batch-size', type=int, default=2000, help='size of each image batch') - parser.add_argument('--printerval', type=int, default=1, help='print results interval') - parser.add_argument('--var', nargs='+', default=[3], help='debug list') + parser.add_argument("--epochs", type=int, default=5000, help="number of epochs") + parser.add_argument("--batch-size", type=int, default=2000, help="size of each image batch") + parser.add_argument("--printerval", type=int, default=1, help="print results interval") + parser.add_argument("--var", nargs="+", default=[3], help="debug list") opt = parser.parse_args() opt.var = [float(x) for x in opt.var] - print(opt, end='\n\n') + print(opt, end="\n\n") init_seeds() if opt.var[0] == 0: - _ = train(H, model=WAVE(), str='.Tanh') + _ = train(H, model=WAVE(), str=".Tanh") elif opt.var[0] == 2: - _ = train(H, model=WAVE2(), str='.Tanh') + _ = train(H, model=WAVE2(), str=".Tanh") elif opt.var[0] == 3: - _ = train(H, model=WAVE3(), str='.Tanh') + _ = train(H, model=WAVE3(), str=".Tanh") elif opt.var[0] == 4: - _ = train(H, model=WAVE4(), str='.Tanh') + _ = train(H, model=WAVE4(), str=".Tanh") # 100K SET --------------------------------------------------------------------- # Model Summary: 8 layers, 33376 parameters, 33376 gradients diff --git a/train_tf.py b/train_tf.py index 751989b..5ae9dbf 100644 --- a/train_tf.py +++ b/train_tf.py @@ -10,8 +10,10 @@ import tensorflow as tf import tensorflow.contrib.eager as tfe + tf.enable_eager_execution() + def runexample(H, model, str): lr = 0.002 eps = 0.001 @@ -19,40 +21,44 @@ def runexample(H, model, str): validations = 5000 printInterval = 1000 # batch_size = 10000 - data = 'wavedata25ns.mat' + data = "wavedata25ns.mat" cuda = tf.test.is_gpu_available() tf.set_random_seed(1) - path = 'data/' - os.makedirs(path + 'models', exist_ok=True) - name = (data[:-4] + '%s%glr%geps%s' % (H[:], lr, eps, str) - ).replace(', ', '_').replace('[', '_').replace(']', '_') + path = "data/" + os.makedirs(path + "models", exist_ok=True) + name = (data[:-4] + "%s%glr%geps%s" % (H[:], lr, eps, str)).replace(", ", "_").replace("[", "_").replace("]", "_") tica = time.time() - device = '/gpu:0' if cuda else '/cpu:0' - print('Running %s on %s' % (name, device)) + device = "/gpu:0" if cuda else "/cpu:0" + print("Running %s on %s" % (name, device)) if not os.path.isfile(path + data): - os.system('wget -P data/ https://storage.googleapis.com/ultralytics/' + data) + os.system("wget -P data/ https://storage.googleapis.com/ultralytics/" + data) mat = scipy.io.loadmat(path + data) - x = mat['inputs'] # inputs (nx512) [waveform1 waveform2] - y = mat['outputs'][:, 0:2] # outputs (nx4) [position(mm), time(ns), PE, E(MeV)] + x = mat["inputs"] # inputs (nx512) [waveform1 waveform2] + y = mat["outputs"][:, 0:2] # outputs (nx4) [position(mm), time(ns), PE, E(MeV)] nz, nx = x.shape ny = y.shape[1] if model is None: # model = WAVE(nx, ny, H) - model = tf.keras.Sequential([ - tf.keras.layers.Dense(H[0], activation=tf.tanh, input_shape=(512,)), # must declare input shape - tf.keras.layers.Dense(H[1], activation=tf.tanh), - tf.keras.layers.Dense(H[2], activation=tf.tanh, ), - tf.keras.layers.Dense(ny), - ]) + model = tf.keras.Sequential( + [ + tf.keras.layers.Dense(H[0], activation=tf.tanh, input_shape=(512,)), # must declare input shape + tf.keras.layers.Dense(H[1], activation=tf.tanh), + tf.keras.layers.Dense( + H[2], + activation=tf.tanh, + ), + tf.keras.layers.Dense(ny), + ] + ) x, _, _ = normalize(x, 1) # normalize each input row y, ymu, ys = normalize(y, 0) # normalize each output column x, y, xv, yv, xt, yt = splitdata(x, y, train=0.70, validate=0.15, test=0.15, shuffle=False) - labels = ['train', 'validate', 'test'] + labels = ["train", "validate", "test"] print(model) @@ -69,10 +75,9 @@ def criteria(y_pred, y): # MSE ticb = time.time() L = np.full((epochs, 3), np.nan) - best = (0, 1E6, None) # best (epoch, validation loss, model) + best = (0, 1e6, None) # best (epoch, validation loss, model) with tf.device(device): for i in range(epochs): - # Calculate derivatives of the input function with respect to its parameters. with tf.GradientTape() as tape: y_pred = model(x) @@ -89,40 +94,40 @@ def criteria(y_pred, y): # MSE if L[i, 1] < best[1]: best = (i, L[i, 1], None) if (i - best[0]) > validations: - print('\n%g validation checks exceeded at epoch %g.' % (validations, i)) + print("\n%g validation checks exceeded at epoch %g." % (validations, i)) break if i % printInterval == 0: # print and save progress # scipy.io.savemat(path + name + '.mat', dict(bestepoch=best[0], loss=L[best[0]], L=L, name=name)) _, std = stdtf(y_predv - yv, ys) - print('%.3fs' % (time.time() - ticb), i, L[i], std) + print("%.3fs" % (time.time() - ticb), i, L[i], std) ticb = time.time() # Apply the gradient to the model optimizer.apply_gradients(zip(grads, model.variables), global_step=tf.train.get_or_create_global_step()) else: - print('WARNING: Validation loss still decreasing after %g epochs (train longer).' % (i + 1)) + print("WARNING: Validation loss still decreasing after %g epochs (train longer)." % (i + 1)) # torch.save(best[2], path + 'models/' + name + '.pt') # model.load_state_dict(best[2]) dt = time.time() - tica - print('\nFinished %g epochs in %.3fs (%.3f epochs/s)\nBest results from epoch %g:' % (i + 1, dt, i / dt, best[0])) + print("\nFinished %g epochs in %.3fs (%.3f epochs/s)\nBest results from epoch %g:" % (i + 1, dt, i / dt, best[0])) loss, std = np.zeros(3), np.zeros((3, ny)) for i, (xi, yi) in enumerate(((x, y), (xv, yv), (xt, yt))): loss[i], std[i] = stdtf(model(xi) - yi, ys) - print('%.5f %s %s' % (loss[i], std[i, :], labels[i])) + print("%.5f %s %s" % (loss[i], std[i, :], labels[i])) # scipy.io.savemat(path + name + '.mat', dict(bestepoch=best[0], loss=loss, std=std, L=L, name=name)) # files.download(path + name + '.mat') data = [] for i, s in enumerate(labels): - data.append(go.Scatter(x=np.arange(epochs), y=L[:, i], mode='markers+lines', name=s)) - layout = go.Layout(xaxis=dict(type='linear', autorange=True), - yaxis=dict(type='log', autorange=True)) - #configure_plotly_browser_state() + data.append(go.Scatter(x=np.arange(epochs), y=L[:, i], mode="markers+lines", name=s)) + layout = go.Layout(xaxis=dict(type="linear", autorange=True), yaxis=dict(type="log", autorange=True)) + # configure_plotly_browser_state() plot(go.Figure(data=data, layout=layout)) -if __name__ == '__main__': + +if __name__ == "__main__": H = [128, 32, 8] for i in range(1): - runexample(H, None, '.' + str(i)) + runexample(H, None, "." + str(i)) diff --git a/utils/torch_utils.py b/utils/torch_utils.py index 52c6426..dc7c09d 100644 --- a/utils/torch_utils.py +++ b/utils/torch_utils.py @@ -9,20 +9,24 @@ def init_seeds(seed=0): def select_device(force_cpu=False): cuda = False if force_cpu else torch.cuda.is_available() - device = torch.device('cuda:0' if cuda else 'cpu') + device = torch.device("cuda:0" if cuda else "cpu") if not cuda: - print('Using CPU') + print("Using CPU") if cuda: - c = 1024 ** 2 # bytes to MB + c = 1024**2 # bytes to MB ng = torch.cuda.device_count() x = [torch.cuda.get_device_properties(i) for i in range(ng)] - print("Using CUDA device0 _CudaDeviceProperties(name='%s', total_memory=%dMB)" % - (x[0].name, x[0].total_memory / c)) + print( + "Using CUDA device0 _CudaDeviceProperties(name='%s', total_memory=%dMB)" + % (x[0].name, x[0].total_memory / c) + ) if ng > 0: # torch.cuda.set_device(0) # OPTIONAL: Set GPU ID for i in range(1, ng): - print(" device%g _CudaDeviceProperties(name='%s', total_memory=%dMB)" % - (i, x[i].name, x[i].total_memory / c)) + print( + " device%g _CudaDeviceProperties(name='%s', total_memory=%dMB)" + % (i, x[i].name, x[i].total_memory / c) + ) return device diff --git a/utils/utils.py b/utils/utils.py index 18aeb91..230e1a0 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -5,8 +5,8 @@ import torch # Set printoptions -torch.set_printoptions(linewidth=1320, precision=5, profile='long') -np.set_printoptions(linewidth=320, formatter={'float_kind': '{:11.5g}'.format}) # format short g, %precision=5 +torch.set_printoptions(linewidth=1320, precision=5, profile="long") +np.set_printoptions(linewidth=320, formatter={"float_kind": "{:11.5g}".format}) # format short g, %precision=5 def normalize(x, axis=None): # normalize x mean and std by axis @@ -38,14 +38,14 @@ def splitdata(x, y, train=0.7, validate=0.15, test=0.15, shuffle=False): # spli def stdpt(r, ys): # MSE loss + standard deviation (pytorch) r = r.detach() - loss = (r ** 2).mean().cpu().item() + loss = (r**2).mean().cpu().item() std = r.std(0).cpu().numpy() * ys return loss, std def stdtf(r, ys): # MSE loss + standard deviation (tf eager) r = r.numpy() - loss = (r ** 2).mean() + loss = (r**2).mean() std = r.std(0) * ys return loss, std @@ -54,12 +54,14 @@ def model_info(model): # Plots a line-by-line description of a PyTorch model n_p = sum(x.numel() for x in model.parameters()) # number parameters n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients - print('\n%5s %40s %9s %12s %20s %10s %10s' % ('layer', 'name', 'gradient', 'parameters', 'shape', 'mu', 'sigma')) + print("\n%5s %40s %9s %12s %20s %10s %10s" % ("layer", "name", "gradient", "parameters", "shape", "mu", "sigma")) for i, (name, p) in enumerate(model.named_parameters()): - name = name.replace('module_list.', '') - print('%5g %40s %9s %12g %20s %10.3g %10.3g' % ( - i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std())) - print('Model Summary: %g layers, %g parameters, %g gradients' % (i + 1, n_p, n_g)) + name = name.replace("module_list.", "") + print( + "%5g %40s %9s %12g %20s %10.3g %10.3g" + % (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std()) + ) + print("Model Summary: %g layers, %g parameters, %g gradients" % (i + 1, n_p, n_g)) class patienceStopper(object): @@ -76,7 +78,7 @@ def __init__(self, patience=10, verbose=True, epochs=1000, printerval=10): self.printerval = printerval def reset(self): - self.bestloss = float('inf') + self.bestloss = float("inf") self.bestmetrics = None self.num_bad_epochs = 0 @@ -99,28 +101,30 @@ def step(self, loss, metrics=None, model=None): self.bestmodel = copy.deepcopy(model) if self.num_bad_epochs > self.patience: - self.final('%g Patience exceeded at epoch %g.' % (self.patience, self.epoch)) + self.final("%g Patience exceeded at epoch %g." % (self.patience, self.epoch)) return True elif self.epoch >= self.epochs: - self.final('WARNING: %g Patience not exceeded by epoch %g (train longer).' % (self.patience, self.epoch)) + self.final("WARNING: %g Patience not exceeded by epoch %g (train longer)." % (self.patience, self.epoch)) return True else: return False def first(self, model): - s = ('epoch', 'time', 'loss', 'metric(s)') - print('%12s' * len(s) % s) + s = ("epoch", "time", "loss", "metric(s)") + print("%12s" * len(s) % s) def printepoch(self, epoch, loss, metrics): s = (epoch, time.time() - self.t, loss) if metrics is not None: for i in range(len(metrics)): s += (metrics[i],) - print('%12.5g' * len(s) % s) + print("%12.5g" * len(s) % s) self.t = time.time() def final(self, msg): dt = time.time() - self.t0 - print('%s\nFinished %g epochs in %.3fs (%.3f epochs/s). Best results:' % ( - msg, self.epochs + 1, dt, (self.epochs + 1) / dt)) + print( + "%s\nFinished %g epochs in %.3fs (%.3f epochs/s). Best results:" + % (msg, self.epochs + 1, dt, (self.epochs + 1) / dt) + ) self.printepoch(self.bestepoch, self.bestloss, self.bestmetrics)