-
-
Notifications
You must be signed in to change notification settings - Fork 44
/
DeadLockDetector.java
168 lines (158 loc) · 6.03 KB
/
DeadLockDetector.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
package me.nallar.tickthreading.minecraft;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.util.EnumSet;
import java.util.Map;
import java.util.TreeMap;
import cpw.mods.fml.common.FMLCommonHandler;
import cpw.mods.fml.common.ITickHandler;
import cpw.mods.fml.common.TickType;
import cpw.mods.fml.common.registry.TickRegistry;
import cpw.mods.fml.relauncher.Side;
import me.nallar.tickthreading.Log;
import net.minecraft.network.packet.Packet3Chat;
import net.minecraft.server.MinecraftServer;
public class DeadLockDetector {
private boolean sentWarningRecently = false;
private static volatile String lastJob = "";
private static volatile long lastTickTime = 0;
private static final ITickHandler tickHandler = new ITickHandler() {
private final EnumSet<TickType> tickTypes = EnumSet.of(TickType.SERVER, TickType.CLIENTGUI);
@Override
public void tickStart(EnumSet<TickType> type, Object... tickData) {
tick("Server tick start");
}
@Override
public void tickEnd(EnumSet<TickType> type, Object... tickData) {
}
@Override
public EnumSet<TickType> ticks() {
return tickTypes;
}
@Override
public String getLabel() {
return "TickThreading Deadlock Detector";
}
};
public DeadLockDetector() {
Thread deadlockThread = new Thread(new Runnable() {
@Override
public void run() {
while (checkForDeadlocks()) {
try {
Thread.sleep(6000);
} catch (InterruptedException ignored) {
}
}
}
});
deadlockThread.setDaemon(true);
deadlockThread.setName("Deadlock Detector");
deadlockThread.start();
TickRegistry.registerTickHandler(tickHandler, Side.SERVER);
TickRegistry.registerTickHandler(tickHandler, Side.CLIENT);
}
public static synchronized long tick(String name) {
lastJob = name;
return lastTickTime = System.currentTimeMillis();
}
public boolean checkForDeadlocks() {
Log.flush();
int deadTime = (int) (System.currentTimeMillis() - lastTickTime);
if (lastTickTime == 0 || !MinecraftServer.getServer().isServerRunning()) {
return true;
}
if (TickThreading.instance.exitOnDeadlock) {
if (sentWarningRecently && deadTime < 10000) {
sentWarningRecently = false;
MinecraftServer.getServerConfigurationManager(MinecraftServer.getServer())
.sendPacketToAllPlayers(new Packet3Chat("The server has recovered and will not need to restart. :)"));
} else if (deadTime > 10000) {
MinecraftServer.getServerConfigurationManager(MinecraftServer.getServer())
.sendPacketToAllPlayers(new Packet3Chat("The server appears to have frozen and will restart soon if it does not recover. :("));
}
}
if (deadTime < (TickThreading.instance.deadLockTime * 1000)) {
return true;
}
if (TickThreading.instance.exitOnDeadlock) {
MinecraftServer.getServerConfigurationManager(MinecraftServer.getServer())
.sendPacketToAllPlayers(new Packet3Chat("The server is restarting - be right back!"));
}
TreeMap<String, Thread> sortedThreads = new TreeMap<String, Thread>();
StringBuilder sb = new StringBuilder();
sb.append("The server appears to have deadlocked.")
.append("\nLast tick ").append(deadTime).append("ms ago.")
.append("\nTicking: ").append(lastJob).append('\n');
Map<Thread, StackTraceElement[]> traces = Thread.getAllStackTraces();
for (Thread thread : traces.keySet()) {
sortedThreads.put(thread.getName(), thread);
}
for (Thread thread : sortedThreads.values()) {
sb.append("Current Thread: ").append(thread.getName()).append('\n').append(" PID: ").append(thread.getId())
.append(" | Alive: ").append(thread.isAlive()).append(" | State: ").append(thread.getState())
.append(" | Daemon: ").append(thread.isDaemon()).append(" | Priority:").append(thread.getPriority())
.append(" Stack:").append('\n');
for (StackTraceElement stackTraceElement : thread.getStackTrace()) {
sb.append(" ").append(stackTraceElement.toString()).append('\n');
}
}
ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
long[] deadlockedThreads = threadMXBean.findDeadlockedThreads();
if (deadlockedThreads != null) {
ThreadInfo[] infos = threadMXBean.getThreadInfo(deadlockedThreads, true, true);
sb.append("Definitely deadlocked: \n");
for (ThreadInfo threadInfo : infos) {
sb.append(threadInfo).append('\n');
}
}
Log.severe(sb.toString());
Log.flush();
// Yes, we save multiple times - handleServerStopping may freeze on the same thing we deadlocked on, but if it doesn't might change stuff
// which needs to be saved.
MinecraftServer minecraftServer = MinecraftServer.getServer();
if (minecraftServer.currentlySaving) {
Log.severe("World state is possibly corrupted! Sleeping for 2 minutes - will force save after.");
Log.flush();
minecraftServer.currentlySaving = false;
try {
Thread.sleep(1000 * 120);
} catch (InterruptedException ignored) {
}
}
Log.info("Attempting to save");
Log.flush();
minecraftServer.saveEverything(); // Save first
new Thread() {
@Override
public void run() {
try {
Thread.sleep(300000);
} catch (InterruptedException ignored) {
}
Log.severe("Froze while attempting to stop - halting server.");
Log.flush();
Runtime.getRuntime().halt(1);
}
}.start();
Log.info("Attempting to stop mods and disconnect players cleanly");
try {
minecraftServer.stopServer();
} catch (Exception e) {
Log.severe("Error stopping server", e);
}
FMLCommonHandler.instance().handleServerStopping(); // Try to get mods to save data - this may lock up, as we deadlocked.
minecraftServer.saveEverything(); // Save again, in case they changed anything.
minecraftServer.initiateShutdown();
Log.flush();
if (TickThreading.instance.exitOnDeadlock) {
try {
Thread.sleep(5000);
} catch (InterruptedException ignored) {
}
Runtime.getRuntime().exit(1);
}
return false;
}
}