-
Notifications
You must be signed in to change notification settings - Fork 4
/
evaluation_baseline_no_attack.py
175 lines (141 loc) · 7.67 KB
/
evaluation_baseline_no_attack.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
"""
assume there is no attack,
simulate the performance where the MTD is triggered by FPR of the data-driven detector
NOTE: the peoriodic MAX RANK and ROBUST MTDs has same performance on the attack and no attack casa
therefore we only simulate the event-trigger condition
"""
from utils.load_data import load_case, load_measurement, load_load_pv, load_dataset
from models.model import LSTM_AE
from models.evaluation import Evaluation
import torch
from configs.config import sys_config, mtd_config, save_metric
from configs.nn_setting import nn_setting
import numpy as np
from tqdm import tqdm
from models.dataset import scaler
from optim.optimization import mtd_optim
from optim.robust_mtd import x_to_b
from evaluation_baseline_with_attack import run_incomplete # incomplete MTD
if __name__ == "__main__":
"""
settings
"""
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--mtd_type", type = str)
total_run = mtd_config['total_run']
args = parser.parse_args()
mtd_type = args.mtd_type
if mtd_type == 'robust':
total_run = mtd_config['total_run']
elif mtd_type == 'max_rank':
total_run = mtd_config['total_run'] * 5
else:
print('Please input the correct mtd_type')
print('mtd type:', mtd_type, 'total run', total_run)
"""
load everything
"""
# Load cases, measurement, and load
case_class = load_case()
z_noise_summary, v_est_summary = load_measurement()
load_active, load_reactive, pv_active_, pv_reactive_ = load_load_pv()
test_dataloader_scaled, test_dataloader_unscaled, valid_dataloader_scaled, valid_dataloader_unscaled = load_dataset()
feature_size = len(z_noise_summary)
test_start_idx = int(feature_size * (nn_setting['train_prop'] + nn_setting['valid_prop'])) # The start index of test dataset
# LSTM-AE detector and identifier
lstm_ae = LSTM_AE()
lstm_ae.load_state_dict(torch.load(nn_setting['model_path'], map_location=torch.device(nn_setting['device'])))
dd_detector = Evaluation(case_class=case_class) # Instance the data-driven detector
scaler_ = scaler() # Instance the scaler class
print(f'Quantile: {dd_detector.quantile[dd_detector.quantile_idx]}')
print(f'Threshold: {dd_detector.ae_threshold[dd_detector.quantile_idx]}')
# Constraints
x_max = case_class.x*(1+mtd_config['x_facts_ratio'])
x_min = case_class.x*(1-mtd_config['x_facts_ratio'])
# Column constraints
if sys_config['case_name'] == 'case14':
degree_one = np.array([8]) - 1 ## buses that are not in any loop
degree_one_posi = []
for i in degree_one:
if i < case_class.ref_index:
degree_one_posi.append(i)
elif i > case_class.ref_index:
degree_one_posi.append(i-1)
col_constraint = 0.999*np.ones((case_class.no_bus-1,))
col_constraint[degree_one_posi] = 1
k_min = 6
# Metrics
x_ratio = []
cost_no_mtd = []
cost_with_mtd = []
posi_trigger = 0 # indicator to detected attack, if equals to 1, then start the next round
for idx_, (idx, input, v_est_pre, v_est_last) in tqdm(enumerate(test_dataloader_unscaled)):
"""
idx_: starts from 0
idx: the actual index number in test_dataloader_unscaled
"""
if idx_ >= int(total_run):
break
if idx >= len(test_dataloader_unscaled.dataset)-1:
# try another one
total_run += 1
continue
if posi_trigger == 1:
# we have already store the result and cost (due to MTD trigger) in the previous iteration
posi_trigger = 0
continue
v_est_last = v_est_last.flatten() # The ground truth state
# true state
vang_true = np.angle(v_est_last.numpy())
# Scale
input_scale = scaler_(input)
encoded, decoded, loss_lattent, loss_recons = dd_detector.evaluate(input_scale) # detect on the normal meausurement
# normal operation without MTD
current_load_idx = test_start_idx + 6 + idx.numpy() # The index of the current load and pv in the entire load and pv dataset
result_no_mtd = case_class.run_opf(load_active = (load_active[current_load_idx]-pv_active_[current_load_idx]), load_reactive = load_reactive[current_load_idx], verbose = False)
cost_no_mtd.append(result_no_mtd['f'])
cost_with_mtd.append(result_no_mtd['f'])
x_ratio.append(np.zeros((case_class.no_brh,)))
if loss_recons < dd_detector.ae_threshold[dd_detector.quantile_idx]:
# Negative detection
pass
else:
# Positive detection
posi_trigger = 1 # do not run the next iteration
# trigger random MTD
next_load_idx = test_start_idx + 6 + 1 + idx.numpy() # The index of the next load and pv in the entire load and pv dataset, this is for evaluating the cost
vang_att = np.angle(v_est_last.numpy()) # As if there is an attack on the phase angle
c_true = (vang_att - vang_true) # Ground truth angle attack vector with reference bus included; this is actually zero.
c_recover_no_ref = np.expand_dims(c_true[case_class.non_ref_index],1) # No reference bus
"""
MTD algorithm
"""
if mtd_type == 'robust':
x_mtd, b_mtd, loss = run_incomplete(v_est_last=v_est_last,
case_class = case_class,
x_max = x_max, x_min = x_min,
col_constraint=col_constraint,
k_min = k_min)
elif mtd_type == 'max_rank':
x_mtd = case_class.random_mtd(x_max_ratio = mtd_config['x_facts_ratio'])
b_mtd = x_to_b(case_class, x_mtd)
else:
print("Run MTD algorithm")
# Instance the mtd_optim class for evaluation
mtd_optim_ = mtd_optim(case_class, v_est_last.numpy(), c_recover_no_ref, varrho_square = 0.01) # Only to instance the class c_recover_no_ref and varrho_square is not used
# Evaluate the MTD without attack
is_converged, x_mtd_change_ratio, residual_normal, cost_no_mtd_, cost_mtd_two_ = mtd_optim_.mtd_metric_no_attack(b_mtd = b_mtd,
load_active = load_active[next_load_idx],
load_reactive = load_reactive[next_load_idx],
pv_active = pv_active_[next_load_idx]
)
# log
cost_no_mtd.append(cost_no_mtd_)
cost_with_mtd.append(cost_mtd_two_)
x_ratio.append(np.abs(x_mtd - case_class.x)/case_class.x)
save_metric(address = f'metric/{sys_config["case_name"]}/metric_{mtd_type}_{mtd_config["mode"]}_no_attack.npy',
x_ratio = x_ratio,
cost_no_mtd = cost_no_mtd,
cost_with_mtd = cost_with_mtd
)