Skip to content

Commit

Permalink
work on automatic instance management continues
Browse files Browse the repository at this point in the history
  • Loading branch information
Randgalt committed Oct 11, 2012
1 parent 92871fe commit 5589106
Show file tree
Hide file tree
Showing 9 changed files with 451 additions and 390 deletions.
Expand Up @@ -16,6 +16,7 @@

package com.netflix.exhibitor.core.automanage;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.netflix.exhibitor.core.Exhibitor;
Expand Down Expand Up @@ -96,10 +97,16 @@ public Boolean call() throws Exception
{
if ( lock.lock(exhibitor.getLog(), Exhibitor.AUTO_INSTANCE_MANAGEMENT_PERIOD_MS / 2, TimeUnit.MILLISECONDS) )
{
ServerList potentialServerList = createPotentialServerList(serverList, statuses, usState.getUs() == null);

exhibitor.getLog().add(ActivityLog.Type.INFO, "Automatic Instance Management will change the server list: " + serverList + " ==> " + potentialServerList);
adjustConfig(potentialServerList.toSpecString(), clusterState.getLeaderHostname());
ServerList potentialServerList = createPotentialServerList(serverList, clusterState.getLiveInstances(), usState.getUs() == null);
if ( potentialServerList.getSpecs().size() == 0 )
{
exhibitor.getLog().add(ActivityLog.Type.INFO, "Automatic Instance Management skipped because new potential server list is empty");
}
else
{
exhibitor.getLog().add(ActivityLog.Type.INFO, "Automatic Instance Management will change the server list: " + serverList + " ==> " + potentialServerList);
adjustConfig(potentialServerList.toSpecString(), clusterState.getLeaderHostname());
}
}
}
finally
Expand All @@ -110,6 +117,38 @@ public Boolean call() throws Exception
return true;
}

@VisibleForTesting
void adjustConfig(final String newSpec, String leaderHostname) throws Exception
{
final InstanceConfig currentConfig = exhibitor.getConfigManager().getConfig();
InstanceConfig newConfig = new InstanceConfig()
{
@Override
public String getString(StringConfigs config)
{
if ( config == StringConfigs.SERVERS_SPEC )
{
return newSpec;
}
return currentConfig.getString(config);
}

@Override
public int getInt(IntConfigs config)
{
return currentConfig.getInt(config);
}
};
if ( exhibitor.getConfigManager().startRollingConfig(newConfig, leaderHostname) )
{
clusterState.clear();
}
else
{
exhibitor.getLog().add(ActivityLog.Type.INFO, "Could not initiate Automatic Instance Management config change. Another process is already making a config change.");
}
}

private ServerList createPotentialServerList(ServerList existingList, List<ServerStatus> statuses, boolean addUsIn)
{
List<ServerSpec> newList = Lists.newArrayList();
Expand Down Expand Up @@ -175,35 +214,4 @@ private List<ServerStatus> getStatuses(ServerList serverList)

return statuses;
}

private void adjustConfig(final String newSpec, String leaderHostname) throws Exception
{
final InstanceConfig currentConfig = exhibitor.getConfigManager().getConfig();
InstanceConfig newConfig = new InstanceConfig()
{
@Override
public String getString(StringConfigs config)
{
if ( config == StringConfigs.SERVERS_SPEC )
{
return newSpec;
}
return currentConfig.getString(config);
}

@Override
public int getInt(IntConfigs config)
{
return currentConfig.getInt(config);
}
};
if ( exhibitor.getConfigManager().startRollingConfig(newConfig, leaderHostname) )
{
clusterState.clear();
}
else
{
exhibitor.getLog().add(ActivityLog.Type.INFO, "Could not initiate Automatic Instance Management config change. Another process is already making a config change.");
}
}
}
Expand Up @@ -55,6 +55,22 @@ boolean isInQuorum()
return (getLeaderHostname() != null);
}

List<ServerStatus> getLiveInstances()
{
List<ServerStatus> live = Lists.newArrayList();
List<ServerStatus> currentStatuses = statuses.get();
for ( ServerStatus status : currentStatuses )
{
InstanceStateTypes type = InstanceStateTypes.fromCode(status.getCode());
if ( type != InstanceStateTypes.DOWN )
{
live.add(status);
}
}

return live;
}

boolean hasDeadInstances()
{
List<ServerStatus> currentStatuses = statuses.get();
Expand Down
Expand Up @@ -190,7 +190,7 @@ public synchronized boolean startRollingConfig(final InstanceConfig newConfig, S
}

InstanceConfig currentConfig = getCollection().getRootConfig();
RollingHostNamesBuilder builder = new RollingHostNamesBuilder(currentConfig, newConfig, exhibitor.getLog(), leaderHostname);
RollingHostNamesBuilder builder = new RollingHostNamesBuilder(currentConfig, newConfig, leaderHostname);

clearAttempts();

Expand Down
Expand Up @@ -18,7 +18,6 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import com.netflix.exhibitor.core.activity.ActivityLog;
import com.netflix.exhibitor.core.state.ServerList;
import java.util.List;
import java.util.Set;
Expand All @@ -27,10 +26,8 @@ class RollingHostNamesBuilder
{
private final List<String> rollingHostNames;

RollingHostNamesBuilder(InstanceConfig rootConfig, InstanceConfig rollingConfig, ActivityLog log, String leaderHostname)
RollingHostNamesBuilder(InstanceConfig rootConfig, InstanceConfig rollingConfig, String leaderHostname)
{
// TODO leaderHostname

ServerList rootServers = new ServerList(rootConfig.getString(StringConfigs.SERVERS_SPEC));
ServerList rollingServers = new ServerList(rollingConfig.getString(StringConfigs.SERVERS_SPEC));

Expand All @@ -39,11 +36,11 @@ class RollingHostNamesBuilder

ImmutableList.Builder<String> builder = ImmutableList.builder();
builder.addAll(newServers); // new servers need to be started first as the others will try to communicate with them. You may have issues if there is more than 1 new server
if ( unchangedServers.contains(leaderHostname) )
if ( (leaderHostname != null) && unchangedServers.contains(leaderHostname) )
{
unchangedServers.remove(leaderHostname);
builder.addAll(unchangedServers); // the servers that are staying in the cluster can be restarted next.
builder.add(leaderHostname); // restart the leader last in the hopes of keeping quorum as long as possible
Set<String> allButLeader = Sets.difference(unchangedServers, Sets.newHashSet(leaderHostname));
builder.addAll(allButLeader); // the servers that are staying in the cluster can be restarted next.
builder.add(leaderHostname); // restart the leader last in the hopes of keeping quorum as long as possible
}
else
{
Expand Down
Expand Up @@ -79,6 +79,17 @@ public void setLeader(boolean leader)
isLeader = leader;
}

@Override
public String toString()
{
return "ServerStatus{" +
"hostname='" + hostname + '\'' +
", code=" + code +
", description='" + description + '\'' +
", isLeader=" + isLeader +
'}';
}

@SuppressWarnings("RedundantIfStatement")
@Override
public boolean equals(Object o)
Expand Down
@@ -0,0 +1,201 @@
/*
* Copyright 2012 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.exhibitor.core.automanage;

import com.netflix.exhibitor.core.activity.ActivityLog;
import com.netflix.exhibitor.core.config.ConfigCollection;
import com.netflix.exhibitor.core.config.ConfigProvider;
import com.netflix.exhibitor.core.config.InstanceConfig;
import com.netflix.exhibitor.core.config.IntConfigs;
import com.netflix.exhibitor.core.config.LoadedInstanceConfig;
import com.netflix.exhibitor.core.config.PropertyBasedInstanceConfig;
import com.netflix.exhibitor.core.config.PseudoLock;
import com.netflix.exhibitor.core.config.RollingConfigState;
import com.netflix.exhibitor.core.config.StringConfigs;
import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

class MockConfigProvider implements ConfigProvider
{
private volatile ConfigCollection collection = new PropertyBasedInstanceConfig(new Properties(), new Properties());

void setConfig(final StringConfigs type, final String value)
{
final InstanceConfig config = collection.getRootConfig();
final InstanceConfig newConfig = new InstanceConfig()
{
@Override
public String getString(StringConfigs configType)
{
if ( configType == type )
{
return value;
}
return config.getString(configType);
}

@Override
public int getInt(IntConfigs configType)
{
return config.getInt(configType);
}
};
collection = new ConfigCollection()
{
@Override
public InstanceConfig getConfigForThisInstance(String hostname)
{
return getRootConfig();
}

@Override
public InstanceConfig getRootConfig()
{
return newConfig;
}

@Override
public InstanceConfig getRollingConfig()
{
return null;
}

@Override
public boolean isRolling()
{
return false;
}

@Override
public RollingConfigState getRollingConfigState()
{
return null;
}
};
}

void setConfig(final IntConfigs type, final int value)
{
final InstanceConfig config = collection.getRootConfig();
final InstanceConfig newConfig = new InstanceConfig()
{
@Override
public String getString(StringConfigs configType)
{
return config.getString(configType);
}

@Override
public int getInt(IntConfigs configType)
{
if ( configType == type )
{
return value;
}
return config.getInt(configType);
}
};
collection = new ConfigCollection()
{
@Override
public InstanceConfig getConfigForThisInstance(String hostname)
{
return getRootConfig();
}

@Override
public InstanceConfig getRootConfig()
{
return newConfig;
}

@Override
public InstanceConfig getRollingConfig()
{
return null;
}

@Override
public boolean isRolling()
{
return false;
}

@Override
public RollingConfigState getRollingConfigState()
{
return null;
}
};
}

@Override
public void start() throws Exception
{
}

@Override
public LoadedInstanceConfig loadConfig() throws Exception
{
return new LoadedInstanceConfig(collection, 0)
{
@Override
public ConfigCollection getConfig()
{
return collection;
}
};
}

@Override
public LoadedInstanceConfig storeConfig(ConfigCollection config, long compareVersion) throws Exception
{
collection = config;
return new LoadedInstanceConfig(config, compareVersion)
{
@Override
public ConfigCollection getConfig()
{
return collection;
}
};
}

@Override
public PseudoLock newPseudoLock() throws Exception
{
return new PseudoLock()
{
@Override
public boolean lock(ActivityLog log, long maxWait, TimeUnit unit) throws Exception
{
return true;
}

@Override
public void unlock() throws Exception
{
}
};
}

@Override
public void close() throws IOException
{
}
}

0 comments on commit 5589106

Please sign in to comment.