Skip to content

CA-115557: networkd: implement enic workaround #1956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 22, 2014
Merged

CA-115557: networkd: implement enic workaround #1956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6d931b
networkd: add Cisco UCS enic workaround check
robhoes Sep 22, 2014
6b6c59c
CA-115557: networkd: implement enic workaround
robhoes Sep 22, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 57 additions & 27 deletions ocaml/network/network_server.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ open D
type context = unit

let config : config_t ref = ref empty_config
let backend_kind = ref Openvswitch

let legacy_management_interface_start () =
try
Expand Down Expand Up @@ -83,6 +84,13 @@ let set_dns_interface _ dbg ~name =
debug "Setting DNS interface to %s" name;
config := {!config with dns_interface = Some name}

(* The enic driver is for Cisco UCS devices. The current driver adds VLAN0 headers
* to all incoming packets, which confuses certain guests OSes. The workaround
* constitutes adding a VLAN0 Linux device to strip those headers again.
*)
let need_enic_workaround () =
!backend_kind = Bridge && List.mem "enic" (Sysfs.list_drivers ())

module Interface = struct
let get_config name =
get_config !config.interface_config default_interface name
Expand Down Expand Up @@ -355,6 +363,17 @@ module Interface = struct
end else
config
in
let config =
if need_enic_workaround () then
List.fold_left (fun accu (name, interface) ->
if (Sysfs.is_physical name && Linux_bonding.get_bond_master_of name = None) || Linux_bonding.is_bond_device name then
(name, interface) :: (Ip.vlan_name name 0, interface) :: accu
else
(name, interface) :: accu
) [] config
else
config
in
debug "** Configuring the following interfaces: %s" (String.concat ", " (List.map (fun (name, _) -> name) config));
let exec f = if conservative then (try f () with _ -> ()) else f () in
List.iter (function (name, ({ipv4_conf; ipv4_gateway; ipv6_conf; ipv6_gateway; ipv4_routes; dns=nameservers,domains; mtu;
Expand All @@ -377,7 +396,6 @@ module Interface = struct
end

module Bridge = struct
let kind = ref Openvswitch
let add_default = ref []

let get_config name =
Expand All @@ -393,23 +411,23 @@ module Bridge = struct
let backend = String.strip String.isspace
(Unixext.string_of_file (Fhs.etcdir ^ "/network.conf")) in
match backend with
| "openvswitch" | "vswitch" -> kind := Openvswitch
| "bridge" -> kind := Bridge
| "openvswitch" | "vswitch" -> backend_kind := Openvswitch
| "bridge" -> backend_kind := Bridge
| backend ->
let error = Printf.sprintf "ERROR: network backend unknown (%s)" backend in
debug "%s" error;
failwith error

let get_bond_links_up _ dbg ~name =
Debug.with_thread_associated dbg (fun () ->
match !kind with
match !backend_kind with
| Openvswitch -> Ovs.get_bond_links_up name
| Bridge -> Proc.get_bond_links_up name
) ()

let get_all _ dbg () =
Debug.with_thread_associated dbg (fun () ->
match !kind with
match !backend_kind with
| Openvswitch -> Ovs.list_bridges ()
| Bridge -> Sysfs.get_all_bridges ()
) ()
Expand All @@ -421,7 +439,7 @@ module Bridge = struct
| Some (parent, vlan) -> Printf.sprintf " (VLAN %d on bridge %s)" vlan parent
);
update_config name {get_config name with vlan; bridge_mac=mac; other_config};
begin match !kind with
begin match !backend_kind with
| Openvswitch ->
let fail_mode =
if not (List.mem_assoc "vswitch-controller-fail-mode" other_config) then
Expand Down Expand Up @@ -496,7 +514,7 @@ module Bridge = struct
let destroy _ dbg ?(force=false) ~name () =
Debug.with_thread_associated dbg (fun () ->
Interface.bring_down () dbg ~name;
match !kind with
match !backend_kind with
| Openvswitch ->
if Ovs.get_vlans name = [] || force then begin
debug "Destroying bridge %s" name;
Expand Down Expand Up @@ -529,8 +547,14 @@ module Bridge = struct
Interface.bring_down () dbg ~name:dev;
if Linux_bonding.is_bond_device dev then
Linux_bonding.remove_bond_master dev;
if String.startswith "eth" dev && String.contains dev '.' then
ignore (Ip.destroy_vlan dev)
if (String.startswith "eth" dev || String.startswith "bond" dev) && String.contains dev '.' then begin
ignore (Ip.destroy_vlan dev);
let n = String.length dev in
if String.sub dev (n - 2) 2 = ".0" && need_enic_workaround () then
let vlan_base = String.sub dev 0 (n - 2) in
if Linux_bonding.is_bond_device vlan_base then
Linux_bonding.remove_bond_master (String.sub dev 0 (n - 2))
end;
) ifs;
Interface.set_ipv4_conf () dbg ~name ~conf:None4;
ignore (Brctl.destroy_bridge name)
Expand All @@ -540,12 +564,12 @@ module Bridge = struct

let get_kind _ dbg () =
Debug.with_thread_associated dbg (fun () ->
!kind
!backend_kind
) ()

let get_ports _ dbg ~name =
Debug.with_thread_associated dbg (fun () ->
match !kind with
match !backend_kind with
| Openvswitch -> Ovs.bridge_to_ports name
| Bridge -> raise Not_implemented
) ()
Expand All @@ -556,14 +580,14 @@ module Bridge = struct
let ports = List.concat (List.map (fun (_, {ports}) -> ports) !config.bridge_config) in
List.map (fun (port, {interfaces}) -> port, interfaces) ports
else
match !kind with
match !backend_kind with
| Openvswitch -> List.concat (List.map Ovs.bridge_to_ports (Ovs.list_bridges ()))
| Bridge -> raise Not_implemented
) ()

let get_bonds _ dbg ~name =
Debug.with_thread_associated dbg (fun () ->
match !kind with
match !backend_kind with
| Openvswitch -> Ovs.bridge_to_ports name
| Bridge -> raise Not_implemented
) ()
Expand All @@ -575,21 +599,21 @@ module Bridge = struct
let names = List.map (fun (port, {interfaces}) -> port, interfaces) ports in
List.filter (fun (_, ifs) -> List.length ifs > 1) names
else
match !kind with
match !backend_kind with
| Openvswitch -> List.concat (List.map Ovs.bridge_to_ports (Ovs.list_bridges ()))
| Bridge -> raise Not_implemented
) ()

let get_vlan _ dbg ~name =
Debug.with_thread_associated dbg (fun () ->
match !kind with
match !backend_kind with
| Openvswitch -> Ovs.bridge_to_vlan name
| Bridge -> raise Not_implemented
) ()

let add_default_flows _ dbg bridge mac interfaces =
Debug.with_thread_associated dbg (fun () ->
match !kind with
match !backend_kind with
| Openvswitch -> Ovs.add_default_flows bridge mac interfaces
| Bridge -> ()
) ()
Expand All @@ -608,7 +632,7 @@ module Bridge = struct
debug "Adding port %s to bridge %s with interfaces %s%s" name bridge
(String.concat ", " interfaces)
(match bond_mac with Some mac -> " and MAC " ^ mac | None -> "");
match !kind with
match !backend_kind with
| Openvswitch ->
if List.length interfaces = 1 then begin
List.iter (fun name -> Interface.bring_up () dbg ~name) interfaces;
Expand All @@ -633,10 +657,9 @@ module Bridge = struct
name bridge
end
| Bridge ->
if List.length interfaces = 1 then begin
List.iter (fun name -> Interface.bring_up () dbg ~name) interfaces;
ignore (Brctl.create_port bridge name)
end else begin
if List.length interfaces = 1 then
List.iter (fun name -> Interface.bring_up () dbg ~name) interfaces
else begin
if not (List.mem name (Sysfs.bridge_to_interfaces bridge)) then begin
Linux_bonding.add_bond_master name;
let bond_properties =
Expand All @@ -652,9 +675,16 @@ module Bridge = struct
| None -> warn "No MAC address specified for the bond"
end
end;
Interface.bring_up () dbg ~name;
Interface.bring_up () dbg ~name
end;
if need_enic_workaround () then begin
debug "Applying enic workaround: adding VLAN0 device to bridge";
Ip.create_vlan name 0;
let vlan0 = Ip.vlan_name name 0 in
Interface.bring_up () dbg ~name:vlan0;
ignore (Brctl.create_port bridge vlan0)
end else
ignore (Brctl.create_port bridge name)
end
) ()

let remove_port _ dbg ~bridge ~name =
Expand All @@ -665,7 +695,7 @@ module Bridge = struct
let ports = List.remove_assoc name config.ports in
update_config bridge {config with ports}
end;
match !kind with
match !backend_kind with
| Openvswitch ->
ignore (Ovs.destroy_port name)
| Bridge ->
Expand All @@ -674,7 +704,7 @@ module Bridge = struct

let get_interfaces _ dbg ~name =
Debug.with_thread_associated dbg (fun () ->
match !kind with
match !backend_kind with
| Openvswitch ->
Ovs.bridge_to_interfaces name
| Bridge ->
Expand All @@ -683,7 +713,7 @@ module Bridge = struct

let get_fail_mode _ dbg ~name =
Debug.with_thread_associated dbg (fun () ->
match !kind with
match !backend_kind with
| Openvswitch ->
begin match Ovs.get_fail_mode name with
| "standalone" -> Some Standalone
Expand Down Expand Up @@ -766,7 +796,7 @@ let on_startup () =
(* the following is best-effort *)
read_config ();
remove_centos_config ();
if !Bridge.kind = Openvswitch then
if !backend_kind = Openvswitch then
Ovs.set_max_idle 5000;
Bridge.make_config () dbg ~conservative:true ~config:!config.bridge_config ();
Interface.make_config () dbg ~conservative:true ~config:!config.interface_config ();
Expand Down
7 changes: 7 additions & 0 deletions ocaml/network/network_utils.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ module Sysfs = struct
let all = Array.to_list (Sys.readdir "/sys/class/net") in
List.filter (fun name -> Sys.is_directory ("/sys/class/net/" ^ name)) all

let list_drivers () =
try
Array.to_list (Sys.readdir "/sys/bus/pci/drivers")
with _ ->
warn "Failed to obtain list of drivers from sysfs";
[]

let getpath dev attr =
Printf.sprintf "/sys/class/net/%s/%s" dev attr

Expand Down