From 32d7efbfb03e1d5ce2c73e4341e1b5949254c0ed Mon Sep 17 00:00:00 2001 From: Rodolfo Alonso Hernandez Date: Mon, 19 Feb 2024 09:38:07 +0000 Subject: [PATCH 1/4] Retry ``set|get_link_attribute(s)`` if the interface is not present After some interface operations (in particular the ``IpLinkCommand.set_ns`` operation), the network interface is temporarily not present in the destination namespace. This patch retries the interface "ip link set|show" command in that case. Related-Bug: #1961740 Change-Id: I5a57cfc71ad59f1fe9ea65e19b1a32314d798729 (cherry picked from commit 016f5187a5503b835cd6803c5d8fe6f05091df12) --- neutron/privileged/agent/linux/ip_lib.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/neutron/privileged/agent/linux/ip_lib.py b/neutron/privileged/agent/linux/ip_lib.py index 6b36ef2d1df..2e913f023e1 100644 --- a/neutron/privileged/agent/linux/ip_lib.py +++ b/neutron/privileged/agent/linux/ip_lib.py @@ -348,6 +348,10 @@ def set_link_flags(device, namespace, flags): _run_iproute_link("set", device, namespace, flags=new_flags) +@tenacity.retry( + retry=tenacity.retry_if_exception_type(NetworkInterfaceNotFound), + wait=tenacity.wait_exponential(multiplier=0.02, max=1), + stop=tenacity.stop_after_delay(3), reraise=True) @privileged.link_cmd.entrypoint def set_link_attribute(device, namespace, **attributes): _run_iproute_link("set", device, namespace, **attributes) @@ -378,7 +382,8 @@ def set_link_bridge_master(device, bridge, namespace=None): @tenacity.retry( retry=tenacity.retry_if_exception_type( - netlink_exceptions.NetlinkDumpInterrupted), + (netlink_exceptions.NetlinkDumpInterrupted, + NetworkInterfaceNotFound)), wait=tenacity.wait_exponential(multiplier=0.02, max=1), stop=tenacity.stop_after_delay(8), reraise=True) From 85debddc1c09176bd9d50af59ee615cd3de78e5c Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Sat, 24 Feb 2024 12:18:45 -0800 Subject: [PATCH 2/4] Remove TripleO jobs TripleO project is retiring so removing its jobs - https://review.opendev.org/c/openstack/governance/+/905145 Change-Id: Id910656d73bb0d55c68612b64f6b710f0955ca26 --- zuul.d/job-templates.yaml | 21 ----------- zuul.d/tripleo.yaml | 79 --------------------------------------- 2 files changed, 100 deletions(-) delete mode 100644 zuul.d/tripleo.yaml diff --git a/zuul.d/job-templates.yaml b/zuul.d/job-templates.yaml index b97857c6275..ee790d7977f 100644 --- a/zuul.d/job-templates.yaml +++ b/zuul.d/job-templates.yaml @@ -70,19 +70,6 @@ voting: true - ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa - openstacksdk-functional-devstack-networking - - tripleo-ci-centos-9-content-provider - - neutron-centos-9-tripleo-standalone: - voting: true - vars: &consumer_vars - consumer_job: true - build_container_images: false - remove_tags: - - build - dependencies: &consumer_deps - - tripleo-ci-centos-9-content-provider - - neutron-ovn-tripleo-ci-centos-9-containers-multinode: - vars: *consumer_vars - dependencies: *consumer_deps - neutron-linuxbridge-tempest-plugin-nftables - neutron-ovs-tempest-plugin-iptables_hybrid-nftables - devstack-enforce-scope @@ -144,14 +131,6 @@ voting: true - ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa - openstacksdk-functional-devstack-networking - - tripleo-ci-centos-9-content-provider - - neutron-centos-9-tripleo-standalone: - voting: true - vars: *consumer_vars - dependencies: *consumer_deps - - neutron-ovn-tripleo-ci-centos-9-containers-multinode: - vars: *consumer_vars - dependencies: *consumer_deps - neutron-linuxbridge-tempest-plugin-scenario-nftables - neutron-ovs-tempest-plugin-scenario-iptables_hybrid-nftables - devstack-enforce-scope diff --git a/zuul.d/tripleo.yaml b/zuul.d/tripleo.yaml deleted file mode 100644 index ecb1a0cdff1..00000000000 --- a/zuul.d/tripleo.yaml +++ /dev/null @@ -1,79 +0,0 @@ -- job: - name: neutron-centos-9-tripleo-standalone - parent: tripleo-ci-centos-9-standalone - voting: false - branches: ^master$ - vars: - featureset: '052' - featureset_override: - standalone_environment_files: - - 'environments/services/neutron-ovs.yaml' - run_tempest: false - tempest_format: container - tempest_run_concurrency: 2 - tempest_private_net_provider_type: 'vxlan' - tempest_tempest_conf_overrides: - auth.use_dynamic_credentials: true - tempest_test_whitelist: - - 'tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops' - use_os_tempest: true - irrelevant-files: - - ^test-requirements.txt$ - - ^releasenotes/.*$ - - ^doc/.*$ - - ^setup.cfg$ - - ^.*\.conf\.sample$ - - ^.*\.rst$ - - ^neutron/locale/.*$ - - ^neutron/tests/unit/.*$ - - ^tools/.*$ - - ^tox.ini$ - - ^neutron/agent/ovn/.*$ - - ^neutron/agent/windows/.*$ - - ^neutron/plugins/ml2/drivers/linuxbridge/.*$ - - ^neutron/plugins/ml2/drivers/macvtap/.*$ - - ^neutron/plugins/ml2/drivers/mech_sriov/.*$ - - ^neutron/plugins/ml2/drivers/ovn/.*$ - - ^vagrant/.*$ - - ^roles/.*functional.*$ - - ^playbooks/.*functional.*$ - - ^zuul.d/(?!(project)).*\.yaml - -- job: - name: neutron-ovn-tripleo-ci-centos-9-containers-multinode - parent: tripleo-ci-centos-9-containers-multinode - branches: ^master$ - vars: - nodes: 1ctlr - featureset: '010' - irrelevant-files: - - ^test-requirements.txt$ - - ^releasenotes/.*$ - - ^doc/.*$ - - ^setup.cfg$ - - ^.*\.conf\.sample$ - - ^.*\.rst$ - - ^neutron/locale/.*$ - - ^neutron/tests/unit/.*$ - - ^tools/.*$ - - ^tox.ini$ - - ^neutron/agent/dhcp/.*$ - - ^neutron/agent/l2/.*$ - - ^neutron/agent/l3/.*$ - - ^neutron/agent/metadata/.*$ - - ^neutron/agent/windows/.*$ - - ^neutron/agent/dhcp_agent.py - - ^neutron/agent/l3_agent.py - - ^neutron/agent/metadata_agent.py - - ^neutron/agent/resource_cache.py - - ^neutron/agent/rpc.py - - ^neutron/agent/securitygroup_rpc.py - - ^neutron/plugins/ml2/drivers/linuxbridge/.*$ - - ^neutron/plugins/ml2/drivers/openvswitch/.*$ - - ^neutron/plugins/ml2/drivers/macvtap/.*$ - - ^neutron/plugins/ml2/drivers/mech_sriov/.*$ - - ^neutron/scheduler/.*$ - - ^roles/.*functional.*$ - - ^playbooks/.*functional.*$ - - ^vagrant/.*$ - - ^zuul.d/(?!(project)).*\.yaml From aedb872e4f29b00d7658faccc0e664e00f2d2613 Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Thu, 22 Feb 2024 10:06:58 +0100 Subject: [PATCH 3/4] Ensure that haproxy spawned by the metadata agents is active In both neutron-metadata and neutron-ovn-metadata agents we should ensure that haproxy service spawned for network/router is actually active before moving on. This patch adds that check and this is similar to what was already implemented some time ago for the dnsmasq process spawned by the dhcp agent. Conflicts: neutron/tests/unit/agent/dhcp/test_agent.py Related-Bug: #2052787 Change-Id: Ic58640d89952fa03bd1059608ee6c9072fbaabf5 (cherry picked from commit 2f7f7c2fc29d0ac26b5ff9d82867952a40f0fa1b) (cherry picked from commit 0dfe8dedd63aba2bf5b75ad8494b0ead4ba1b79f) --- neutron/agent/metadata/driver.py | 2 +- neutron/agent/ovn/metadata/driver.py | 2 +- neutron/tests/unit/agent/dhcp/test_agent.py | 3 ++- neutron/tests/unit/agent/metadata/test_driver.py | 7 ++++++- neutron/tests/unit/agent/ovn/metadata/test_driver.py | 7 ++++++- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/neutron/agent/metadata/driver.py b/neutron/agent/metadata/driver.py index 5f1d6190fee..4b5b5279a33 100644 --- a/neutron/agent/metadata/driver.py +++ b/neutron/agent/metadata/driver.py @@ -295,7 +295,7 @@ def spawn_monitored_metadata_proxy(cls, monitor, ns_name, port, conf, ns_name=ns_name, callback=callback) try: - pm.enable() + pm.enable(ensure_active=True) except exceptions.ProcessExecutionError as exec_err: LOG.error("Encountered process execution error %(err)s while " "starting process in namespace %(ns)s", diff --git a/neutron/agent/ovn/metadata/driver.py b/neutron/agent/ovn/metadata/driver.py index 5b59635914f..56f302c80f6 100644 --- a/neutron/agent/ovn/metadata/driver.py +++ b/neutron/agent/ovn/metadata/driver.py @@ -206,7 +206,7 @@ def spawn_monitored_metadata_proxy(cls, monitor, ns_name, port, conf, ns_name=ns_name, callback=callback) try: - pm.enable() + pm.enable(ensure_active=True) except exceptions.ProcessExecutionError as exec_err: LOG.error("Encountered process execution error %(err)s while " "starting process in namespace %(ns)s", diff --git a/neutron/tests/unit/agent/dhcp/test_agent.py b/neutron/tests/unit/agent/dhcp/test_agent.py index 7fa4b1dbd9b..44a88a9fc0f 100644 --- a/neutron/tests/unit/agent/dhcp/test_agent.py +++ b/neutron/tests/unit/agent/dhcp/test_agent.py @@ -637,6 +637,7 @@ def test_dhcp_ready_ports_updates_after_enable_dhcp(self, *args): 'IpAddrCommand.wait_until_address_ready') as mock_wait: mock_wait.return_value = True dhcp = dhcp_agent.DhcpAgent(HOSTNAME) + dhcp.update_isolated_metadata_proxy = mock.Mock() self.assertEqual(set(), dhcp.dhcp_ready_ports) dhcp.configure_dhcp_for_network(fake_network) self.assertEqual({fake_port1.id}, dhcp.dhcp_ready_ports) @@ -834,7 +835,7 @@ def _enable_dhcp_helper(self, network, enable_isolated_metadata=False, process_instance.assert_has_calls([ mock.call.disable(sig=str(int(signal.SIGTERM))), mock.call.get_pid_file_name(), - mock.call.enable()]) + mock.call.enable(ensure_active=True)]) else: process_instance.assert_has_calls([ mock.call.disable(sig=str(int(signal.SIGTERM)))]) diff --git a/neutron/tests/unit/agent/metadata/test_driver.py b/neutron/tests/unit/agent/metadata/test_driver.py index aad6cf36a23..70e6f867758 100644 --- a/neutron/tests/unit/agent/metadata/test_driver.py +++ b/neutron/tests/unit/agent/metadata/test_driver.py @@ -168,7 +168,12 @@ def _test_spawn_metadata_proxy(self, dad_failed=False): 'IpAddrCommand.wait_until_address_ready') as mock_wait,\ mock.patch( 'neutron.agent.linux.ip_lib.' - 'delete_ip_address') as mock_del: + 'delete_ip_address') as mock_del,\ + mock.patch( + 'neutron.agent.linux.external_process.' + 'ProcessManager.active', + new_callable=mock.PropertyMock, + side_effect=[False, True]): agent = l3_agent.L3NATAgent('localhost') agent.process_monitor = mock.Mock() cfg_file = os.path.join( diff --git a/neutron/tests/unit/agent/ovn/metadata/test_driver.py b/neutron/tests/unit/agent/ovn/metadata/test_driver.py index c6c8227ed48..e790fdd7382 100644 --- a/neutron/tests/unit/agent/ovn/metadata/test_driver.py +++ b/neutron/tests/unit/agent/ovn/metadata/test_driver.py @@ -68,7 +68,12 @@ def test_spawn_metadata_proxy(self): return_value=test_utils.FakeUser(self.EUNAME)),\ mock.patch('grp.getgrnam', return_value=test_utils.FakeGroup(self.EGNAME)),\ - mock.patch('os.makedirs'): + mock.patch('os.makedirs'),\ + mock.patch( + 'neutron.agent.linux.external_process.' + 'ProcessManager.active', + new_callable=mock.PropertyMock, + side_effect=[False, True]): cfg_file = os.path.join( metadata_driver.HaproxyConfigurator.get_config_path( cfg.CONF.state_path), From b7e219391d85f9c7651868f498b00627dacef354 Mon Sep 17 00:00:00 2001 From: elajkat Date: Tue, 27 Feb 2024 18:51:40 +0100 Subject: [PATCH 4/4] Stable-Only: Temporary stop running grenade jobs on Zed Due to the change of stable/yoga to unmaintained/yoga grenade jobs are failing on stable/zed. We temporary disabling these jobs, also removes the skip level job from periodic queue, but keeps as it is in experimental queue. Once the issue is fixed we can re enable these jobs. Change-Id: I18b825c8ba7cfb38d82fd259b74a02877eb839d1 Related-Bug: #2052915 --- zuul.d/job-templates.yaml | 1 - zuul.d/project.yaml | 13 +++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/zuul.d/job-templates.yaml b/zuul.d/job-templates.yaml index ee790d7977f..674a91e10b3 100644 --- a/zuul.d/job-templates.yaml +++ b/zuul.d/job-templates.yaml @@ -126,7 +126,6 @@ - neutron-ovn-tempest-ovs-release-ipv6-only - neutron-ovs-tempest-fips - neutron-ovn-tempest-ovs-release-fips - - neutron-ovs-grenade-multinode-tick-tick - devstack-tobiko-neutron: voting: true - ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index a21e99ca811..8238dacfd9d 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -20,8 +20,11 @@ - neutron-functional-with-uwsgi - neutron-fullstack-with-uwsgi - neutron-ovs-rally-task - - neutron-ovs-grenade-multinode - - neutron-ovs-grenade-dvr-multinode + # TODO: re enable once jobs fixed + #- neutron-ovs-grenade-multinode: + # voting: false + #- neutron-ovs-grenade-dvr-multinode: + # voting: false - neutron-ovs-tempest-multinode-full - neutron-ovs-tempest-dvr-ha-multinode-full - neutron-ovn-tempest-ipv6-only @@ -32,6 +35,8 @@ - neutron-functional-with-uwsgi - neutron-fullstack-with-uwsgi - neutron-ovs-tempest-multinode-full - - neutron-ovs-grenade-multinode - - neutron-ovs-grenade-dvr-multinode + #- neutron-ovs-grenade-multinode: + # voting: false + #- neutron-ovs-grenade-dvr-multinode: + # voting: false - neutron-ovn-tempest-ipv6-only