Permalink
Switch branches/tags
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
542 lines (432 sloc) 24.2 KB
# -*- coding: utf-8 -*-
require 'isono'
require 'ipaddress'
require 'tmpdir'
module Dcmgr
module NodeModules
class ServiceNetfilter < Isono::NodeModules::Base
include Dcmgr::Logger
include Dcmgr::Helpers::NicHelper
include Dcmgr::EdgeNetworking
include Dcmgr::EdgeNetworking::Netfilter
initialize_hook do
@worker_thread = Isono::ThreadPool.new(1, 'Netfilter')
event = Isono::NodeModules::EventChannel.new(node)
sleep 1
if Dcmgr::Configurations.hva.verbose_netfilter_cache
@cache = CacheDumper.new(NetfilterCache.new(node), STDOUT)
else
@cache = NetfilterCache.new(node)
end
@task_manager = TaskManagerFactory.create_task_manager(node)
@task_manager.netfilter_hook_script_path = Dcmgr::Configurations.hva.netfilter_hook_script_path
@worker_thread.pass {
@cache.update
#flush all netfilter rules
cmds = []
cmds.concat [
"iptables -t nat -F",
"iptables -t nat -X",
"iptables -t nat -Z",
"iptables -t filter -F",
"iptables -t filter -X",
"iptables -t filter -Z",
"iptables -t raw -F",
"iptables -t raw -X",
"iptables -t raw -Z",
] if Dcmgr::Configurations.hva.enable_iptables
cmds.concat [
"ebtables -t nat --init-table",
"ebtables -t filter --init-table",
] if Dcmgr::Configurations.hva.enable_ebtables
puts cmds.join("\n") if Dcmgr::Configurations.hva.verbose_netfilter
system(cmds.join("\n"))
if script = Dcmgr::Configurations.hva.netfilter_script_post_flush
logger.info("Applying iptables/ebtables rules from: #{script}")
system(script)
end
if Dcmgr::Configurations.hva.debug_iptables
@task_manager.apply_tasks([Dcmgr::EdgeNetworking::Tasks::DebugIptables.new])
end
(@cache.get_all_local_vnics + @cache.get_all_empty_vnics).each { |vif|
friends = @cache.get_all_friends(vif[:uuid])
security_groups = vif[:security_groups].empty? ? [] : @cache.get_security_groups_of_local_vnic(vif[:uuid])
logger.info "initializing vnic '#{vif[:uuid]}'"
@task_manager.apply_vnic_chains(vif)
@task_manager.apply_vnic_tasks(vif,@cache.get_vnic_network_mode(vif[:uuid]).netfilter_all_tasks(vif,@cache.get_vnic_network(vif[:uuid]),friends,security_groups,node))
myinstance.subscribe_to_local_vnic(vif[:uuid])
}
@cache.get_all_security_groups.each { |group|
myinstance.subscribe_to_security_group(group[:uuid])
group[:referencees].each { |ref_id,ref|
myinstance.subscribe_to_referencees(ref_id)
}
}
}
myinstance.subscribe_to_event("hva.#{myinstance.node.manifest.node_instance_id}/vnic_created") do |args|
@worker_thread.pass {
vnic_id = args[0]
logger.debug "event caught: hva.#{myinstance.node.manifest.node_instance_id}/vnic_created: #{vnic_id}"
# Store the security group ids that were in the cache before adding this vnic.
# We need this to fix a bug further down.
old_security_group_ids = @cache.get_all_security_groups.map {|sg| sg[:uuid] }
@cache.add_vnic(vnic_id)
vnic = @cache.get_local_vnic(vnic_id)
if vnic
# This is a fix for the following bug.
# When starting an instance with multiple vnics in the same security group,
# some vnics would get their isolation rules applied twice because if there
# were no vnics in said security group before this instance was started.
#
# We delete the other vnics from the cache so their isolation rules don't get
# applied yet. Another "vnic_created" event for them will be handled shortly
# after this one. The isolation rules will properly be applied at that time.
new_groups_added = vnic[:security_groups] - old_security_group_ids
new_groups_added.each { |group_id|
@cache.get_local_vnics_in_group(group_id).each { |other_vnic|
@cache.remove_local_vnic_from_group(other_vnic[:uuid],group_id) if (vnic[:instance_uuid] == other_vnic[:instance_uuid]) && (vnic[:uuid] != other_vnic[:uuid])
}
}
friends = @cache.get_all_friends(vnic_id)
security_groups = @cache.get_security_groups_of_local_vnic(vnic_id)
else
logger.debug "hva.#{myinstance.node.manifest.node_instance_id}/vnic_created/#{vnic_id}: vnic has no security groups."
vnic = @cache.get_empty_vnic(vnic_id)
friends = []
security_groups = []
end
@task_manager.apply_vnic_chains(vnic)
@task_manager.apply_vnic_tasks(vnic,@cache.get_vnic_network_mode(vnic[:uuid]).netfilter_all_tasks(vnic,@cache.get_vnic_network(vnic[:uuid]),friends,security_groups,node))
# Add isolation tasks to friend vnics
@cache.get_all_local_friends(vnic[:uuid]).each { |friend|
@task_manager.apply_vnic_tasks(friend,@cache.get_vnic_network_mode(friend[:uuid]).netfilter_isolation_tasks(friend,[vnic],node))
}
# Subscribe to the vnic's security groups' event queues
vnic[:security_groups].each { |secg_id|
myinstance.subscribe_to_security_group(secg_id)
secg = @cache.get_group(secg_id)
secg[:referencees].each { |ref_id,ref|
myinstance.subscribe_to_referencees(ref_id)
}
}
# Subscribe to the events for live security group changing
myinstance.subscribe_to_local_vnic(vnic[:uuid])
}
end
myinstance.subscribe_to_event("hva.#{myinstance.node.manifest.node_instance_id}/vnic_destroyed") do |args|
@worker_thread.pass {
vnic_id = args[0]
logger.debug "event caught: hva.#{myinstance.node.manifest.node_instance_id}/vnic_destroyed: #{vnic_id}"
vif = @cache.get_local_vnic(vnic_id) || @cache.get_empty_vnic(vnic_id)
if vif.nil?
logger.debug "vnic not found in cache: #{vnic_id}"
next
end
# Unsubscribe from this vnic's security groups if there aren't any other vnics in them
vif[:security_groups].each { |group_id|
unless @cache.other_local_vnics_left_in_group?(vnic_id,group_id)
myinstance.unsubscribe_from_security_group(group_id)
# Unsubscribe from referenced security groups
group = @cache.get_group(group_id)
group && group[:referencees].values.each { |ref_id,ref|
# Don't unsubscribe if there are other security groups referencing this group
next if @cache.other_groups_referencing_group?(group_id,ref_id)
myinstance.unsubscribe_from_referenced_security_group(ref_id)
}
end
}
myinstance.unsubscribe_from_local_vnic(vif[:uuid])
#Clean up the isolation tasks in friends' chains
if @cache.vnic_has_ip?(vnic_id)
local_friends = @cache.get_all_local_friends(vnic_id)
local_friends.each { |friend|
next unless @cache.vnic_has_ip?(friend[:uuid])
@task_manager.remove_vnic_tasks(friend,@cache.get_vnic_network_mode(friend[:uuid]).netfilter_isolation_tasks(friend,[vif],self.node))
}
# Removing the nat tasks separately because they include an arp reply
# that isn't put in a separate chain. All other tasks are automatically
# deleted when their chains are removed
if @cache.vnic_is_natted?(vnic_id)
all_friends = @cache.get_all_friends(vnic_id)
@task_manager.remove_vnic_tasks(vif,@cache.get_vnic_network_mode(vif[:uuid]).netfilter_isolation_tasks(vif,all_friends,self.node))
end
# Removing this outside of the is_natted? check because it includes
# the address translation for metadata server.
network = @cache.get_network(vif[:network_id])
@task_manager.remove_vnic_tasks(vif, @cache.get_vnic_network_mode(vif[:uuid]).netfilter_nat_tasks(vif,network,self.node) )
end
@task_manager.remove_vnic_chains(vif)
@cache.remove_vnic(vnic_id)
vif[:security_groups].each { |group_id|
@cache.remove_security_group(group_id) unless @cache.other_local_vnics_left_in_group?(vnic_id,group_id)
}
@cache.remove_network(vif[:network_id]) unless @cache.local_vnics_left_in_network?(vif[:network_id]) || @cache.empty_vnics_left_in_network?(vif[:network_id])
}
end
myinstance.subscribe_to_event('broadcast/debug/vnet') do |args|
@worker_thread.pass {
case args[0]
when 'switch_info'
result = { :type => 'netfilter' }
event.publish('gather/debug/vnet', { :sender => args[1], :args => [args[0], myinstance.node.node_id, result] })
end
}
end
end
def subscribe_to_event(ev,&blk)
@subscribed_events = [] if @subscribed_events.nil?
unless @subscribed_events.member?(ev)
event.subscribe(ev,'#',&blk)
@subscribed_events << ev
logger.debug("Subscribing to: #{ev}")
end
end
def unsubscribe_from_event(ev)
if @subscribed_events.member?(ev)
logger.debug("Unsubscribing from: #{ev}")
@subscribed_events.delete(ev)
event.unsubscribe(ev)
end
end
def unsubscribe_from_local_vnic(vnic_id)
unsubscribe_from_event("#{vnic_id}/joined_group")
unsubscribe_from_event("#{vnic_id}/left_group")
end
def subscribe_to_local_vnic(vnic_id)
# This event happens when a running instance joined a security group
subscribe_to_event("#{vnic_id}/joined_group") do |args|
@worker_thread.pass {
group_id = args[0]
logger.debug "event caught: #{vnic_id}/joined_group: #{group_id}"
@cache.add_vnic_to_security_group(vnic_id,group_id)
group_map = @cache.get_group(group_id)
raise "failed to add #{vnic_id} to #{group_id}" unless @cache.is_local_vnic_in_group?(vnic_id,group_id)
vnic_map = @cache.get_local_vnic(vnic_id)
unless vnic_map.nil?
@task_manager.apply_vnic_tasks(vnic_map,@cache.get_vnic_network_mode(vnic_map[:uuid]).netfilter_secgroup_tasks(vnic_map, group_map))
local_friends = @cache.get_local_friends_in_group(vnic_id,group_id)
# Add isolation rules to local friends
local_friends.each { |local_vnic_map|
@task_manager.apply_vnic_tasks(local_vnic_map,@cache.get_vnic_network_mode(local_vnic_map[:uuid]).netfilter_isolation_tasks(local_vnic_map,[vnic_map],self.node))
}
# Add isolation rules to self
friends = @cache.get_friends_in_group(vnic_id,group_id)
@task_manager.apply_vnic_tasks(vnic_map,@cache.get_vnic_network_mode(vnic_map[:uuid]).netfilter_isolation_tasks(vnic_map,friends,self.node))
end
subscribe_to_security_group(group_id)
group_map[:referencees].each { |ref_id,ref|
subscribe_to_referencees(ref_id)
}
}
end
# This event happens when a running instance left a security group
subscribe_to_event("#{vnic_id}/left_group") do |args|
@worker_thread.pass {
group_id = args[0]
logger.debug "event caught: #{vnic_id}/left_group: #{group_id}"
vnic_map = @cache.get_local_vnic(vnic_id)
group_map = @cache.get_group(group_id)
unless vnic_map.nil?
@task_manager.remove_vnic_tasks(vnic_map,@cache.get_vnic_network_mode(vnic_map[:uuid]).netfilter_secgroup_tasks(vnic_map, group_map))
local_friends = @cache.get_local_friends_in_group(vnic_id,group_id)
# Remove isolation rules from local friends
local_friends.each { |local_vnic_map|
@task_manager.remove_vnic_tasks(local_vnic_map,@cache.get_vnic_network_mode(local_vnic_map[:uuid]).netfilter_isolation_tasks(local_vnic_map,[vnic_map],self.node))
}
# Remove isolation rules from self
# Do not remove isolation rules if the vnics are still in another group together
friends = @cache.get_friends_in_group(vnic_id,group_id)
@task_manager.remove_vnic_tasks(vnic_map,@cache.get_vnic_network_mode(vnic_map[:uuid]).netfilter_isolation_tasks(vnic_map,friends,self.node))
#Remove security group from the cache
@cache.remove_local_vnic_from_group(vnic_id,group_id)
@cache.remove_vnic_from_referencees(group_id,vnic_id)
@cache.remove_security_group(group_id) unless @cache.local_vnics_left_in_group?(group_id)
end
# Unsubscribe from "#{ref_group_id}/rules_updated, referencer_added, referencer_left" if this is the last local vnic in the group
# Unsubscribe from "#{ref_group_id}/vnic_added, vnic_left" if this is the last local vnic in the group AND no references are made to the group
if @cache.get_local_vnics_in_group(group_id).empty?
@cache.remove_security_group(group_id)
unsubscribe_from_security_group(group_id)
if @cache.get_local_groups_that_reference_group(group_id).empty?
unsubscribe_from_referenced_security_group(group_id)
end
end
@cache.remove_security_group(group_id) unless @cache.local_vnics_left_in_group?(group_id)
}
end
end
def subscribe_to_referencees(ref_group_id)
subscribe_to_event("#{ref_group_id}/vnic_joined") do |args|
@worker_thread.pass {
vnic_id = args[0]
logger.debug "event caught: #{ref_group_id}/vnic_joined: #{vnic_id}"
was_foreign_vnic_in_group = @cache.is_foreign_vnic_in_group?(vnic_id,ref_group_id)
# Apply isolation tasks for this new vnic to its friends
if @cache.is_local_group?(ref_group_id)
@cache.add_vnic_to_security_group(vnic_id,ref_group_id) unless @cache.is_local_vnic_in_group?(vnic_id,ref_group_id)
end
@cache.add_vnic_to_referencers_and_referencees_for_group(vnic_id,ref_group_id)
unless @cache.is_foreign_vnic_in_group?(vnic_id,ref_group_id) || @cache.is_local_vnic?(vnic_id) || @cache.is_referencee_vnic_in_group?(vnic_id,ref_group_id)
raise "Unknown vnic: #{vnic_id}"
end
# We only need to add isolation rules if it's a foreign vnic
# Local vnics handle their isolation rules the moment they are created
if @cache.is_foreign_vnic_in_group?(vnic_id,ref_group_id) && (not was_foreign_vnic_in_group)
foreign_vnic_map = @cache.get_foreign_vnic(vnic_id,ref_group_id)
local_friends = @cache.get_local_friends_in_group(vnic_id,ref_group_id)
local_friends.each { |local_vnic_map|
@task_manager.apply_vnic_tasks(local_vnic_map,@cache.get_vnic_network_mode(local_vnic_map[:uuid]).netfilter_isolation_tasks(local_vnic_map,[foreign_vnic_map],self.node))
}
end
# If any local security groups are referencing this group, we need to add ARP rules to them
@cache.get_all_security_groups.each { |secg_map|
ref = @cache.get_referencer(secg_map[:uuid],ref_group_id)
next if ref.nil?
lvnics = @cache.get_local_vnics_in_group(ref_group_id)
referencer_vnic_map = ref[vnic_id]
lvnics.each { |lvnic|
@task_manager.apply_vnic_tasks(lvnic,@cache.get_vnic_network_mode(lvnic[:uuid]).netfilter_arp_isolation_tasks(lvnic,[referencer_vnic_map],node))
}
}
add_vnic_to_ref_group(ref_group_id,vnic_id)
}
end
subscribe_to_event("#{ref_group_id}/vnic_left") do |args|
@worker_thread.pass {
vnic_id = args[0]
logger.debug "event caught: #{ref_group_id}/vnic_left: #{vnic_id}"
# Isolation rules if we have local vnics in this group
if @cache.is_foreign_vnic_in_group?(vnic_id,ref_group_id)
foreign_vnic_map = @cache.get_foreign_vnics_in_group(ref_group_id).find {|v| v[:uuid] == vnic_id}
local_friends = @cache.get_local_friends_in_group(vnic_id,ref_group_id)
local_friends.each { |local_vnic_map|
@task_manager.remove_vnic_tasks(local_vnic_map,@cache.get_vnic_network_mode(local_vnic_map[:uuid]).netfilter_isolation_tasks(local_vnic_map,[foreign_vnic_map],self.node))
}
end
remove_vnic_from_ref_group(ref_group_id,vnic_id)
@cache.remove_foreign_vnic(ref_group_id,vnic_id)
@cache.remove_vnic_from_referencees(ref_group_id,vnic_id)
@cache.remove_vnic_from_referencers(ref_group_id,vnic_id)
}
end
end
def subscribe_to_security_group(group_id)
subscribe_to_referencees(group_id)
subscribe_to_event("#{group_id}/referencer_added") do |args|
@worker_thread.pass {
ref_group_id = args[0]
logger.debug "event caught: #{group_id}/referencer_added: #{ref_group_id}"
@cache.update_referencers(group_id)
@cache.update_referencees(group_id)
group = @cache.get_group(group_id)
referencer = @cache.get_referencer(group_id,ref_group_id)
# Open a hole in the ARP wall for the new referencer's vnics
@cache.get_local_vnics_in_group(group_id).each { |lvnic|
@task_manager.apply_vnic_tasks(lvnic,@cache.get_vnic_network_mode(lvnic[:uuid]).netfilter_arp_isolation_tasks(lvnic,referencer.values,node))
}
subscribe_to_referencees(group_id)
}
end
subscribe_to_event("#{group_id}/referencer_removed") do |args|
@worker_thread.pass {
ref_group_id = args[0]
logger.debug "event caught: #{group_id}/referencer_removed: #{ref_group_id}"
group = @cache.get_group(group_id)
referencer = @cache.get_referencer(group_id,ref_group_id)
# Close the ARP wall for the old referencer's vnics
@cache.get_local_vnics_in_group(group_id).each { |lvnic|
@task_manager.remove_vnic_tasks(lvnic,@cache.get_vnic_network_mode(lvnic[:uuid]).netfilter_arp_isolation_tasks(lvnic,referencer.values,node))
}
@cache.update_referencers(group_id)
@cache.update_referencees(group_id)
# Unsubscribe from groups that are no longer referenced
# Don't unsubscribe if there are local vnics in the group or if there are other local security groups referencing this group
unless @cache.local_referencers_left?(ref_group_id) || @cache.local_vnics_left_in_group?(ref_group_id)
unsubscribe_from_referenced_security_group(ref_group_id)
end
}
end
subscribe_to_event("#{group_id}/rules_updated") do |args|
@worker_thread.pass {
logger.debug "event caught: #{group_id}/rules_updated"
logger.info "updating security group '#{group_id}'"
# Determine the difference between the old and new rules
old_group = @cache.get_group(group_id)
old_referencee_vnics = @cache.get_referencee_vnics(group_id)
old_referencee_uuids = @cache.get_group(group_id)[:referencees].keys
@cache.update_rules(group_id)
@cache.update_referencees(group_id)
updated_group = @cache.get_group(group_id)
updated_referencee_vnics = @cache.get_referencee_vnics(group_id)
updated_referencee_uuids = @cache.get_group(group_id)[:referencees].keys
@cache.get_local_vnics_in_group(group_id).each { |vnic|
# Add the new security group tasks
@task_manager.apply_vnic_tasks(vnic, @cache.get_vnic_network_mode(vnic[:uuid]).netfilter_arp_isolation_tasks(vnic,updated_referencee_vnics,node))
@task_manager.apply_vnic_tasks(vnic, @cache.get_vnic_network_mode(vnic[:uuid]).netfilter_secgroup_tasks(vnic, updated_group))
# Remove the old security group tasks
@task_manager.remove_vnic_tasks(vnic, @cache.get_vnic_network_mode(vnic[:uuid]).netfilter_arp_isolation_tasks(vnic,old_referencee_vnics,node))
@task_manager.remove_vnic_tasks(vnic, @cache.get_vnic_network_mode(vnic[:uuid]).netfilter_secgroup_tasks(vnic, old_group))
}
# Subscribe to new referenced groups
updated_referencee_uuids.each { |ref_group_id|
subscribe_to_referencees(ref_group_id) unless old_referencee_uuids.member?(ref_group_id)
}
# Unsubscribe from groups that are no longer referenced
old_referencee_uuids.each { |ref_group_id|
# Don't unsubscribe if it's still referencered
next if updated_referencee_uuids.member?(ref_group_id)
# Don't unsubscribe if there are local vnics in the group
next unless @cache.local_vnics_left_in_group?(ref_group_id)
# Don't unsubscribe if there are other security groups referencing this group
next unless @cache.local_referencers_left?(ref_group_id)
unsubscribe_from_referenced_security_group(ref_group_id)
}
}
end
end
def unsubscribe_from_referenced_security_group(group_id)
unsubscribe_from_event("#{group_id}/vnic_joined")
unsubscribe_from_event("#{group_id}/vnic_left")
end
def unsubscribe_from_security_group(group_id)
unsubscribe_from_referenced_security_group(group_id) unless @cache.local_referencees_left?(group_id)
unsubscribe_from_event("#{group_id}/referencer_added")
unsubscribe_from_event("#{group_id}/referencer_removed")
unsubscribe_from_event("#{group_id}/rules_updated")
end
def add_vnic_to_ref_group(group_id,vnic_id)
handle_ref_group_vnic(group_id,vnic_id,:apply)
end
def remove_vnic_from_ref_group(group_id,vnic_id)
handle_ref_group_vnic(group_id,vnic_id,:remove)
end
def handle_ref_group_vnic(group_id,vnic_id,action = :apply)
#If this is a referenced group, remove the reference rules only for this vnic
@cache.get_all_security_groups.each { |secg_map|
dummy_group = secg_map[:referencees][group_id]
# Don't do anything if this group isn't referencing group_id
next if dummy_group.nil?
vnics = @cache.get_local_vnics_in_group(secg_map[:uuid])
# Create a dummy security group that only contains the rules for this new vnic
# Delete the other rules in the dummy group so that only the new vnic is applied
dummy_group = secg_map
dummy_group[:rules].delete_if { |rule| rule[:ip_source] != group_id }
dummy_group[:referencees][group_id].delete_if { |key,value| key != vnic_id }
referencer_vnic_map = dummy_group[:referencees][group_id][vnic_id]
vnics.each { |vnic_map|
# If the added vnic is local, then its rules are already applied. Don't reapply them
next if vnic_map[:uuid] == vnic_id
@task_manager.send("#{action}_vnic_tasks", vnic_map, @cache.get_vnic_network_mode(vnic_map[:uuid]).netfilter_arp_isolation_tasks(vnic_map,[referencer_vnic_map],node))
@task_manager.send("#{action}_vnic_tasks", vnic_map, @cache.get_vnic_network_mode(vnic_map[:uuid]).netfilter_secgroup_tasks(vnic_map, dummy_group))
}
}
end
def rpc
@rpc ||= Isono::NodeModules::RpcChannel.new(@node)
end
def event
@event ||= Isono::NodeModules::EventChannel.new(@node)
end
end
end
end